backend/smart_fusion.py

"""
Smart Fusion - Document relevance scoring and ranking
Fiel al app original Next.js
"""

import re
from typing import List, Dict, Any

# Default weights (from original)
DEFAULT_SESSION_WEIGHT = 1000
DEFAULT_PREVIOUS_QUERY_WEIGHT = 100
DEFAULT_TITLE_MATCH_WEIGHT = 50
DEFAULT_SNIPPET_MATCH_WEIGHT = 10
DEFAULT_MIN_THRESHOLD = 30
DEFAULT_TOP_N = 150


def normalize_text(text: str) -> str:
    """Remove diacritics for cross-language matching."""
    text = text.lower()
    accents = {'á':'a','é':'e','í':'i','ó':'o','ú':'u','ñ':'n','Á':'A','É':'E','Í':'I','Ó':'O','Ú':'U','Ñ':'N'}
    return re.sub(r'[áéíóúñÁÉÍÓÚÑ]', lambda m: accents.get(m.group(), m.group()), text)


def get_tokens(text: str) -> set:
    """Extract meaningful tokens from text."""
    text = normalize_text(text)
    tokens = set(re.findall(r'[a-záéíóúñ]{4,}', text))
    stopwords = {'para', 'como', 'más', 'pero', 'desde', 'hasta', 'sobre', 'entre', 'the', 'and', 'with', 'from', 'that', 'this', 'have', 'been', 'were', 'they'}
    return tokens - stopwords


def score_document(doc: dict, query: str, is_session: bool = True,
                   session_weight: int = DEFAULT_SESSION_WEIGHT,
                   previous_query_weight: int = DEFAULT_PREVIOUS_QUERY_WEIGHT,
                   title_weight: int = DEFAULT_TITLE_MATCH_WEIGHT,
                   snippet_weight: int = DEFAULT_SNIPPET_MATCH_WEIGHT) -> int:
    """Score a document against a query."""
    score = 0
    
    if is_session:
        score += session_weight
    
    # Title token matching
    title_tokens = get_tokens(doc.get("title", ""))
    query_tokens = get_tokens(query)
    title_matches = len(title_tokens & query_tokens)
    score += title_matches * title_weight
    
    # Snippet matching
    snippet_tokens = get_tokens(doc.get("snippet", "") or doc.get("abstract", ""))
    snippet_matches = len(snippet_tokens & query_tokens)
    score += snippet_matches * snippet_weight
    
    # Query match in stored queries
    stored_queries = doc.get("metadata", {}).get("queries", []) if doc.get("metadata") else []
    for sq in stored_queries:
        if query.lower() in sq.lower():
            score += previous_query_weight
            break
    
    # Non-session docs need minimum matches
    if not is_session and title_matches < 1 and snippet_matches < 4:
        score = 0
    
    return score


def smart_fusion_rank(docs: list, query: str, weights: dict = None) -> list:
    """Rank and filter documents using smart fusion scoring."""
    w = weights or {}
    
    scored = []
    for doc in docs:
        is_session = doc.get("_isSession", True)
        score = score_document(
            doc, query, is_session,
            session_weight=w.get("sessionWeight", DEFAULT_SESSION_WEIGHT),
            previous_query_weight=w.get("previousQueryWeight", DEFAULT_PREVIOUS_QUERY_WEIGHT),
            title_weight=w.get("titleMatchWeight", DEFAULT_TITLE_MATCH_WEIGHT),
            snippet_weight=w.get("snippetMatchWeight", DEFAULT_SNIPPET_MATCH_WEIGHT)
        )
        doc["smartFusionScore"] = score
        scored.append(doc)
    
    # Filter by threshold
    threshold = w.get("minThreshold", DEFAULT_MIN_THRESHOLD)
    filtered = [d for d in scored if d["smartFusionScore"] >= threshold or d.get("_isSession")]
    
    # Sort by score descending
    filtered.sort(key=lambda x: x["smartFusionScore"], reverse=True)
    
    # Cap at top N
    top_n = w.get("topN", DEFAULT_TOP_N)
    return filtered[:top_n]


def merge_with_memory(new_docs: list, stored_records: list, query: str) -> list:
    """Merge new search results with existing memory records."""
    record_map = {r.get("title", "").lower().strip(): r for r in stored_records}
    title_map = {r.get("title", "").lower().strip(): r for r in stored_records}
    
    merged = list(stored_records)
    
    for doc in new_docs:
        title_key = doc.get("title", "").lower().strip()
        if title_key in title_map:
            # Update existing record
            existing = title_map[title_key]
            if doc.get("snippet") and not existing.get("snippet"):
                existing["snippet"] = doc["snippet"]
            if doc.get("pdfUrl") and not existing.get("pdfUrl"):
                existing["pdfUrl"] = doc["pdfUrl"]
        else:
            # Add new record
            merged.append(doc)
            title_map[title_key] = doc
    
    return merged