""" Smart Fusion - Document relevance scoring and ranking Fiel al app original Next.js """ import re from typing import List, Dict, Any # Default weights (from original) DEFAULT_SESSION_WEIGHT = 1000 DEFAULT_PREVIOUS_QUERY_WEIGHT = 100 DEFAULT_TITLE_MATCH_WEIGHT = 50 DEFAULT_SNIPPET_MATCH_WEIGHT = 10 DEFAULT_MIN_THRESHOLD = 30 DEFAULT_TOP_N = 150 def normalize_text(text: str) -> str: """Remove diacritics for cross-language matching.""" text = text.lower() accents = {'á':'a','é':'e','í':'i','ó':'o','ú':'u','ñ':'n','Á':'A','É':'E','Í':'I','Ó':'O','Ú':'U','Ñ':'N'} return re.sub(r'[áéíóúñÁÉÍÓÚÑ]', lambda m: accents.get(m.group(), m.group()), text) def get_tokens(text: str) -> set: """Extract meaningful tokens from text.""" text = normalize_text(text) tokens = set(re.findall(r'[a-záéíóúñ]{4,}', text)) stopwords = {'para', 'como', 'más', 'pero', 'desde', 'hasta', 'sobre', 'entre', 'the', 'and', 'with', 'from', 'that', 'this', 'have', 'been', 'were', 'they'} return tokens - stopwords def score_document(doc: dict, query: str, is_session: bool = True, session_weight: int = DEFAULT_SESSION_WEIGHT, previous_query_weight: int = DEFAULT_PREVIOUS_QUERY_WEIGHT, title_weight: int = DEFAULT_TITLE_MATCH_WEIGHT, snippet_weight: int = DEFAULT_SNIPPET_MATCH_WEIGHT) -> int: """Score a document against a query.""" score = 0 if is_session: score += session_weight # Title token matching title_tokens = get_tokens(doc.get("title", "")) query_tokens = get_tokens(query) title_matches = len(title_tokens & query_tokens) score += title_matches * title_weight # Snippet matching snippet_tokens = get_tokens(doc.get("snippet", "") or doc.get("abstract", "")) snippet_matches = len(snippet_tokens & query_tokens) score += snippet_matches * snippet_weight # Query match in stored queries stored_queries = doc.get("metadata", {}).get("queries", []) if doc.get("metadata") else [] for sq in stored_queries: if query.lower() in sq.lower(): score += previous_query_weight break # Non-session docs need minimum matches if not is_session and title_matches < 1 and snippet_matches < 4: score = 0 return score def smart_fusion_rank(docs: list, query: str, weights: dict = None) -> list: """Rank and filter documents using smart fusion scoring.""" w = weights or {} scored = [] for doc in docs: is_session = doc.get("_isSession", True) score = score_document( doc, query, is_session, session_weight=w.get("sessionWeight", DEFAULT_SESSION_WEIGHT), previous_query_weight=w.get("previousQueryWeight", DEFAULT_PREVIOUS_QUERY_WEIGHT), title_weight=w.get("titleMatchWeight", DEFAULT_TITLE_MATCH_WEIGHT), snippet_weight=w.get("snippetMatchWeight", DEFAULT_SNIPPET_MATCH_WEIGHT) ) doc["smartFusionScore"] = score scored.append(doc) # Filter by threshold threshold = w.get("minThreshold", DEFAULT_MIN_THRESHOLD) filtered = [d for d in scored if d["smartFusionScore"] >= threshold or d.get("_isSession")] # Sort by score descending filtered.sort(key=lambda x: x["smartFusionScore"], reverse=True) # Cap at top N top_n = w.get("topN", DEFAULT_TOP_N) return filtered[:top_n] def merge_with_memory(new_docs: list, stored_records: list, query: str) -> list: """Merge new search results with existing memory records.""" record_map = {r.get("title", "").lower().strip(): r for r in stored_records} title_map = {r.get("title", "").lower().strip(): r for r in stored_records} merged = list(stored_records) for doc in new_docs: title_key = doc.get("title", "").lower().strip() if title_key in title_map: # Update existing record existing = title_map[title_key] if doc.get("snippet") and not existing.get("snippet"): existing["snippet"] = doc["snippet"] if doc.get("pdfUrl") and not existing.get("pdfUrl"): existing["pdfUrl"] = doc["pdfUrl"] else: # Add new record merged.append(doc) title_map[title_key] = doc return merged