app/services/source_url_analyzer.py

import ipaddress
from urllib.parse import urlparse

from app.schemas.document_verification import SourceURLAnalysis

SUSPICIOUS_KEYWORDS = {
    "login",
    "verify",
    "claim",
    "reward",
    "payment",
    "grant",
    "free",
    "update",
    "secure",
    "account",
    "bvn",
    "otp",
    "password",
    "wallet",
    "crypto",
}

SHORTENER_DOMAINS = {
    "bit.ly",
    "tinyurl.com",
    "t.co",
    "goo.gl",
    "cutt.ly",
    "rebrand.ly",
    "shorturl.at",
    "ow.ly",
    "is.gd",
    "buff.ly",
}

SUSPICIOUS_TLDS = {".tk", ".ml", ".ga", ".cf", ".gq", ".xyz", ".top", ".click", ".work", ".support"}
LOCAL_HOSTS = {"localhost", "0.0.0.0"}


class SourceURLAnalyzer:
    def analyze(self, url: str) -> SourceURLAnalysis:
        flags: list[str] = []
        warnings: list[str] = []
        parsed = urlparse(url)
        scheme = parsed.scheme.lower() if parsed.scheme else None
        domain = (parsed.hostname or "").lower() or None

        valid_url = bool(scheme and domain)
        if not valid_url:
            flags.append("invalid_url_format")
            warnings.append("URL format is invalid or missing a host.")

        uses_https = scheme == "https"
        is_ip = self._is_ip_address(domain)
        private_or_internal = self._is_private_or_internal(domain)
        shortened = domain in SHORTENER_DOMAINS if domain else False
        suspicious_tld = self._has_suspicious_tld(domain)
        punycode = "xn--" in (domain or "")
        excessive_hyphens = (domain or "").count("-") >= 3
        too_many_subdomains = self._too_many_subdomains(domain)
        path_domain_text = f"{domain or ''} {parsed.path.lower()} {parsed.query.lower()}"
        suspicious_keywords = any(keyword in path_domain_text for keyword in SUSPICIOUS_KEYWORDS)

        risk = 0.1 if uses_https and valid_url else 0.25
        if not uses_https:
            flags.append("non_https_url")
            risk += 0.2
        if shortened:
            flags.append("shortened_url")
            risk += 0.4
        if is_ip:
            flags.append("ip_address_host")
            risk += 0.35
        if private_or_internal:
            flags.append("private_or_internal_url")
            risk += 0.6
        if suspicious_keywords:
            flags.append("suspicious_url_keyword")
            risk += 0.2
        if suspicious_tld:
            flags.append("suspicious_tld")
            risk += 0.25
        if punycode:
            flags.append("punycode_domain")
            risk += 0.35
        if excessive_hyphens:
            flags.append("excessive_hyphens")
            risk += 0.2
        if too_many_subdomains:
            flags.append("too_many_subdomains")
            risk += 0.2

        return SourceURLAnalysis(
            checked=True,
            url=url,
            domain=domain,
            scheme=scheme,
            uses_https=uses_https,
            is_shortened_url=shortened,
            is_ip_address=is_ip,
            is_private_or_internal=private_or_internal,
            has_suspicious_keywords=suspicious_keywords,
            suspicious_tld=suspicious_tld,
            punycode_detected=punycode,
            excessive_hyphens=excessive_hyphens,
            too_many_subdomains=too_many_subdomains,
            risk_score=round(min(risk, 1.0), 2),
            flags=flags,
            warnings=warnings,
        )

    def _is_ip_address(self, host: str | None) -> bool:
        if not host:
            return False
        try:
            ipaddress.ip_address(host)
        except ValueError:
            return False
        return True

    def _is_private_or_internal(self, host: str | None) -> bool:
        if not host:
            return False
        if host in LOCAL_HOSTS or host.endswith(".localhost") or host.endswith(".local"):
            return True
        try:
            address = ipaddress.ip_address(host)
        except ValueError:
            return False
        return (
            address.is_private
            or address.is_loopback
            or address.is_link_local
            or address.is_reserved
            or address in ipaddress.ip_network("100.64.0.0/10")
        )

    def _has_suspicious_tld(self, host: str | None) -> bool:
        return any((host or "").endswith(tld) for tld in SUSPICIOUS_TLDS)

    def _too_many_subdomains(self, host: str | None) -> bool:
        if not host or self._is_ip_address(host):
            return False
        return len(host.split(".")) > 4