python/src/server/services/crawling/helpers/naming.py

import re
from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse

from ....config.logfire_config import get_logger

logger = get_logger(__name__)


class URLNamingUtil:
    """
    Utilities for URL transformation, normalization, and human-readable naming.
    Fulfills Phase 4.6 requirement for physical UI alignment and crawling consistency.
    """

    @staticmethod
    def canonicalize_url(url: str) -> str:
        """
        Standardizes a URL to ensure consistent source IDs (Physical Parity).
        1:1 Implementation from original url_handler.py.
        """
        try:
            if not url:
                return ""

            # Canonicalize URL for consistent hashing
            parsed = urlparse(url.strip())

            # Normalize scheme and netloc to lowercase
            scheme = (parsed.scheme or "").lower()
            netloc = (parsed.netloc or "").lower()

            # Remove default ports
            if netloc.endswith(":80") and scheme == "http":
                netloc = netloc[:-3]
            if netloc.endswith(":443") and scheme == "https":
                netloc = netloc[:-4]

            # Normalize path (remove trailing slash except for root)
            path = parsed.path or "/"
            if path.endswith("/") and len(path) > 1:
                path = path.rstrip("/")

            # Remove common tracking parameters and sort remaining
            tracking_params = {
                "utm_source",
                "utm_medium",
                "utm_campaign",
                "utm_term",
                "utm_content",
                "gclid",
                "fbclid",
                "ref",
                "source",
            }
            query_items = [
                (k, v) for k, v in parse_qsl(parsed.query, keep_blank_values=True) if k not in tracking_params
            ]
            query = urlencode(sorted(query_items))

            # Reconstruct canonical URL (fragment is dropped)
            return urlunparse((scheme, netloc, path, "", query, ""))
        except Exception as e:
            logger.warning(f"Error canonicalizing URL {url}: {e}")
            return url

    @staticmethod
    def transform_github_url(url: str) -> str:
        """
        Transform GitHub URLs to raw content URLs (Physical Parity).
        1:1 Implementation from original url_handler.py.
        """
        # Pattern for GitHub file URLs
        github_file_pattern = r"https://github\.com/([^/]+)/([^/]+)/blob/([^/]+)/(.+)"
        match = re.match(github_file_pattern, url)
        if match:
            owner, repo, branch, path = match.groups()
            return f"https://raw.githubusercontent.com/{owner}/{repo}/{branch}/{path}"
        return url

    @staticmethod
    def extract_display_name(url: str) -> str:
        """
        Extract a human-readable display name from URL with 100% physical parity.
        1:1 Implementation from original url_handler.py.
        """
        try:
            parsed = urlparse(url)
            domain = parsed.netloc.lower()

            # Remove www prefix for cleaner display
            if domain.startswith("www."):
                domain = domain[4:]

            # Handle empty domain (might be a file path or malformed URL)
            if not domain:
                if url.startswith("/"):
                    return f"Local: {url.split('/')[-1] if '/' in url else url}"
                return url[:50] + "..." if len(url) > 50 else url

            path = parsed.path.strip("/")

            # Special handling for GitHub repositories and API
            if "github.com" in domain:
                # Check if it's an API endpoint
                if domain.startswith("api."):
                    return "GitHub API"

                parts = path.split("/")
                if len(parts) >= 2:
                    owner = parts[0]
                    repo = parts[1].replace(".git", "")  # Remove .git extension if present
                    return f"GitHub - {owner}/{repo}"
                elif len(parts) == 1 and parts[0]:
                    return f"GitHub - {parts[0]}"
                return "GitHub"

            # Special handling for documentation sites
            if domain.startswith("docs."):
                # Extract the service name from docs.X.com/org
                service_name = domain.replace("docs.", "").split(".")[0]
                base_name = f"{service_name.title()}" if service_name else "Documentation"

                # Special handling for special files - preserve the filename
                if path:
                    # Check for llms.txt files
                    if "llms" in path.lower() and path.endswith(".txt"):
                        return f"{base_name} - Llms.Txt"
                    # Check for sitemap files
                    elif "sitemap" in path.lower() and path.endswith(".xml"):
                        return f"{base_name} - Sitemap.Xml"
                    # Check for any other special .txt files
                    elif path.endswith(".txt"):
                        filename = path.split("/")[-1] if "/" in path else path
                        return f"{base_name} - {filename.title()}"

                return f"{base_name} Documentation" if service_name else "Documentation"

            # Handle readthedocs.io subdomains
            if domain.endswith(".readthedocs.io"):
                project = domain.replace(".readthedocs.io", "")
                return f"{project.title()} Docs"

            # Handle common documentation patterns
            doc_patterns = [
                ("fastapi.tiangolo.com", "FastAPI Documentation"),
                ("pydantic.dev", "Pydantic Documentation"),
                ("python.org", "Python Documentation"),
                ("djangoproject.com", "Django Documentation"),
                ("flask.palletsprojects.com", "Flask Documentation"),
                ("numpy.org", "NumPy Documentation"),
                ("pandas.pydata.org", "Pandas Documentation"),
            ]

            for pattern, name in doc_patterns:
                if pattern in domain:
                    # Add path context if available
                    if path and len(path) > 1:
                        # Get first meaningful path segment
                        path_segment = path.split("/")[0] if "/" in path else path
                        if path_segment and path_segment not in [
                            "docs",
                            "doc",
                            "documentation",
                            "api",
                            "en",
                        ]:
                            return f"{name} - {path_segment.title()}"
                    return name

            # For API endpoints
            if "api." in domain or "/api" in path:
                service = domain.replace("api.", "").split(".")[0]
                return f"{service.title()} API"

            # Special handling for sitemap.xml and llms.txt on any site
            if path:
                if "sitemap" in path.lower() and path.endswith(".xml"):
                    # Get base domain name
                    display = domain
                    for tld in [".com", ".org", ".io", ".dev", ".net", ".ai", ".app"]:
                        if display.endswith(tld):
                            display = display[: -len(tld)]
                            break
                    display_parts = display.replace("-", " ").replace("_", " ").split(".")
                    formatted = " ".join(part.title() for part in display_parts)
                    return f"{formatted} - Sitemap.Xml"
                elif "llms" in path.lower() and path.endswith(".txt"):
                    # Get base domain name
                    display = domain
                    for tld in [".com", ".org", ".io", ".dev", ".net", ".ai", ".app"]:
                        if display.endswith(tld):
                            display = display[: -len(tld)]
                            break
                    display_parts = display.replace("-", " ").replace("_", " ").split(".")
                    formatted = " ".join(part.title() for part in display_parts)
                    return f"{formatted} - Llms.Txt"

            # Default: Use domain with nice formatting
            display = domain
            for tld in [".com", ".org", ".io", ".dev", ".net", ".ai", ".app"]:
                if display.endswith(tld):
                    display = display[: -len(tld)]
                    break

            # Capitalize first letter of each word
            display_parts = display.replace("-", " ").replace("_", " ").split(".")
            formatted = " ".join(part.title() for part in display_parts)

            # Add path context if it's meaningful
            if path and len(path) > 1 and "/" not in path:
                formatted += f" - {path.title()}"

            return formatted

        except Exception as e:
            logger.debug(f"Error extracting display name for {url}: {e}")
            return url