api/utils.py

import hashlib
import re
import logging
from urllib.parse import urlparse

logger = logging.getLogger(__name__)


def get_namespace_id(url: str) -> str:
    """
    Generates a collision-resistant namespace ID.
    Format: clean_name_MD5hash
    """
    clean_name = re.sub(r"https?://(www\.)?", "", url)
    clean_name = re.sub(r"[^a-zA-Z0-9]", "_", clean_name)
    clean_name = clean_name.strip("_")[:30]

    url_hash = hashlib.md5(url.encode("utf-8")).hexdigest()[:6]

    return f"{clean_name}_{url_hash}"


def is_valid_url(url: str, base_domain: str) -> bool:
    """
    Enforces strict crawl scope.
    """
    try:
        parsed = urlparse(url)
        base_parsed = urlparse(base_domain)

        # 1. Scheme Check
        if parsed.scheme not in ["http", "https"]:
            return False

        # 2. Strict Domain Check (Ends with pattern to prevent 'google.com.evil.com')
        # We allow subdomains (e.g. portal.batstateu.edu.ph)
        if not parsed.netloc.endswith(base_parsed.netloc):
            return False

        # 3. File Extension Check
        ignored_exts = [
            ".pdf",
            ".jpg",
            ".png",
            ".gif",
            ".css",
            ".js",
            ".docx",
            ".xlsx",
            ".xml",
            ".zip",
            ".rar",
            ".mp4",
        ]
        if any(parsed.path.lower().endswith(ext) for ext in ignored_exts):
            return False

        return True
    except Exception:
        return False