python/src/server/services/search/keyword_extractor.py

"""
Keyword Extraction Utility

Simple and effective keyword extraction for improved search capabilities.
Uses lightweight Python string operations without heavy NLP dependencies.
"""

import re

from .dictionaries import PRESERVE_KEYWORDS, STOP_WORDS, TECHNICAL_STOP_WORDS


class KeywordExtractor:
    """Simple keyword extraction for search queries"""

    def __init__(self):
        self.stop_words = STOP_WORDS | TECHNICAL_STOP_WORDS
        self.preserve_keywords = PRESERVE_KEYWORDS

    def extract_keywords(self, query: str, min_length: int = 2, max_keywords: int = 10) -> list[str]:
        """
        Extract meaningful keywords from a search query.

        Args:
            query: The search query string
            min_length: Minimum keyword length (default: 2)
            max_keywords: Maximum number of keywords to return (default: 10)

        Returns:
            List of extracted keywords, ordered by importance
        """
        # Convert to lowercase for processing
        query_lower = query.lower()

        # Step 1: Extract potential keywords (alphanumeric + some special chars)
        # Keep dashes and underscores as they're common in tech terms
        tokens = re.findall(r"[a-z0-9_-]+", query_lower)

        # Step 2: Filter tokens
        keywords = []
        for token in tokens:
            # Skip if too short
            if len(token) < min_length:
                continue

            # Always keep if in preserve list
            if token in self.preserve_keywords:
                keywords.append(token)
            # Skip if in stop words
            elif token not in self.stop_words:
                keywords.append(token)

        # Step 3: Handle special cases and compound terms
        # Look for common patterns like "best practices", "how to", etc.
        compound_patterns = [
            (r"best\s+practice[s]?", "best_practices"),
            (r"how\s+to", "howto"),
            (r"step\s+by\s+step", "step_by_step"),
            (r"real\s+time", "realtime"),
            (r"full\s+text", "fulltext"),
            (r"full[\s-]?stack", "fullstack"),
            (r"back[\s-]?end", "backend"),
            (r"front[\s-]?end", "frontend"),
            (r"data[\s-]?base", "database"),
            (r"web[\s-]?socket", "websocket"),
        ]

        for pattern, replacement in compound_patterns:
            if re.search(pattern, query_lower):
                keywords.append(replacement)

        # Step 4: Deduplicate while preserving order
        seen = set()
        unique_keywords = []
        for keyword in keywords:
            if keyword not in seen:
                seen.add(keyword)
                unique_keywords.append(keyword)

        # Step 5: Prioritize keywords
        # - Original case-sensitive matches get priority
        # - Technical terms get priority
        # - Longer terms often more specific
        prioritized = self._prioritize_keywords(unique_keywords, query)

        # Return top N keywords
        return prioritized[:max_keywords]

    def _prioritize_keywords(self, keywords: list[str], original_query: str) -> list[str]:
        """
        Prioritize keywords based on various factors.

        Args:
            keywords: List of extracted keywords
            original_query: The original search query

        Returns:
            Keywords sorted by priority
        """
        keyword_scores = []

        for keyword in keywords:
            score = 0

            # Bonus for exact case match in original
            if keyword in original_query:
                score += 3

            # Bonus for being a known technical term
            if keyword in self.preserve_keywords:
                score += 2

            # Bonus for longer terms (more specific)
            if len(keyword) > 5:
                score += 1

            # Bonus for containing numbers (versions, etc.)
            if any(c.isdigit() for c in keyword):
                score += 1

            # Check if it appears multiple times (important term)
            count = original_query.lower().count(keyword)
            if count > 1:
                score += (count - 1) * 2  # Give more weight to repeated terms

            keyword_scores.append((keyword, score))

        # Sort by score (descending) then by original order
        keyword_scores.sort(key=lambda x: (-x[1], keywords.index(x[0])))

        return [kw for kw, _ in keyword_scores]

    def build_search_terms(self, keywords: list[str]) -> list[str]:
        """
        Build search terms from keywords, including variations.

        Args:
            keywords: List of keywords

        Returns:
            List of search terms including variations
        """
        search_terms = []

        for keyword in keywords:
            # Add the keyword itself
            search_terms.append(keyword)

            # Add plural/singular variations for common patterns
            if keyword.endswith("s") and len(keyword) > 3 and not keyword.endswith("ss"):
                # Possible plural -> add singular (but not for words ending in ss)
                search_terms.append(keyword[:-1])
            elif not keyword.endswith("s") or keyword.endswith("ss"):
                # Possible singular -> add plural
                # Handle special cases
                if keyword.endswith("ss"):
                    search_terms.append(keyword + "es")  # e.g., "class" -> "classes"
                elif keyword.endswith("s"):
                    search_terms.append(keyword + "es")  # Other words ending in s
                else:
                    search_terms.append(keyword + "s")

            # Add common variations
            if keyword.endswith("ing"):
                # Remove -ing
                base = keyword[:-3]
                if len(base) > 2:
                    search_terms.append(base)
                    search_terms.append(base + "e")  # e.g., "coding" -> "code"

            if keyword.endswith("ed"):
                # Remove -ed
                base = keyword[:-2]
                if len(base) > 2:
                    search_terms.append(base)
                    search_terms.append(base + "e")  # e.g., "created" -> "create"

        # Deduplicate
        seen = set()
        unique_terms = []
        for term in search_terms:
            if term not in seen:
                seen.add(term)
                unique_terms.append(term)

        return unique_terms


# Global instance for easy access
keyword_extractor = KeywordExtractor()


def extract_keywords(query: str, min_length: int = 2, max_keywords: int = 10) -> list[str]:
    """
    Convenience function to extract keywords from a query.

    Args:
        query: The search query string
        min_length: Minimum keyword length
        max_keywords: Maximum number of keywords to return

    Returns:
        List of extracted keywords
    """
    return keyword_extractor.extract_keywords(query, min_length, max_keywords)


def build_search_terms(keywords: list[str]) -> list[str]:
    """
    Convenience function to build search terms from keywords.

    Args:
        keywords: List of keywords

    Returns:
        List of search terms including variations
    """
    return keyword_extractor.build_search_terms(keywords)