backend/tools/search_engine.py

import asyncio
from typing import List, Dict, Any, Optional
from backend.providers.openalex import search_openalex
from backend.providers.semantic_scholar import search_semantic_scholar
from backend.providers.pubmed import search_pubmed
from backend.providers.arxiv import search_arxiv
from backend.providers.crossref import search_crossref
from backend.providers.dblp import search_dblp
from backend.providers.latam_repositories import search_alicia, search_la_referencia, search_bDTD, search_rraae
from backend.providers.scopus import search_scopus
from backend.providers.zenodo import search_zenodo
from backend.providers.openaire import search_openaire
from backend.providers.doaj import search_doaj
from backend.providers.core_ import search_core
from backend.providers.redalyc import search_redalyc
from backend.providers.serpapi import search_serpapi
from backend.providers.sources import SOURCE_GROUPS, SOURCE_ALIASES
from modules.config.sources_config_tab import get_enabled_sources

# Provider map
PROVIDERS = {
    "openalex": search_openalex,
    "semantic": search_semantic_scholar,
    "pubmed": search_pubmed,
    "arxiv": search_arxiv,
    "crossref": search_crossref,
    "dblp": search_dblp,
    "alicia": search_alicia,
    "renati": search_alicia,  # Alias to ALICIA as fallback since RENATI API is blocked
    "lareferencia": search_la_referencia,
    "bdtd": search_bDTD,
    "rraae": search_rraae,
    "scopus": search_scopus,
    "zenodo": search_zenodo,
    "openaire": search_openaire,
    "doaj": search_doaj,
    "core": search_core,
    "redalyc": search_redalyc,
    "serpapi": search_serpapi,
}


def expand_sources(sources: List[str]) -> List[str]:
    """Expand source groups and aliases to individual source IDs."""
    expanded = set()
    for s in sources:
        s_lower = s.lower().strip()
        aliased = SOURCE_ALIASES.get(s_lower, s_lower)
        if aliased in SOURCE_GROUPS:
            for src in SOURCE_GROUPS[aliased]:
                expanded.add(src)
        else:
            expanded.add(aliased)
    return list(expanded)


def deduplicate_results(results: List[dict]) -> List[dict]:
    """Deduplicate by DOI and normalized title."""
    seen = {}
    for r in results:
        doi = (r.get("doi") or "").lower().strip()
        if doi:
            key = f"doi:{doi}"
            if key in seen:
                continue
            seen[key] = r
            continue
        title = (r.get("title") or "").lower().strip()[:60]
        if title:
            key = f"title:{title}"
            if key in seen:
                continue
            seen[key] = r
    return list(seen.values())


async def search(query: str, sources: List[str] = None, max_results: int = 50,
                 year_start: str = None, year_end: str = None,
                 enabled_sources: Optional[List[str]] = None) -> dict:
    """Search across multiple academic sources."""
    if not sources:
        sources = ["latam", "global"]

    expanded = expand_sources(sources)
    
    # If no explicit enabled_sources passed, use the global config
    if enabled_sources is None:
        enabled_sources = get_enabled_sources()
    if enabled_sources:
        expanded = [src for src in expanded if src in enabled_sources]

    tasks = []

    for src in expanded:
        if src in PROVIDERS:
            tasks.append(PROVIDERS[src](query, limit=min(max_results, 50)))

    # Run all searches in parallel
    raw_results = await asyncio.gather(*tasks, return_exceptions=True)

    # Flatten and filter exceptions
    all_results = []
    for r in raw_results:
        if isinstance(r, list):
            all_results.extend(r)

    # Apply year filters
    if year_start:
        all_results = [r for r in all_results if r.get("year") and int(r.get("year", 0)) >= int(year_start)]
    if year_end:
        all_results = [r for r in all_results if r.get("year") and int(r.get("year", 0)) <= int(year_end)]

    # Fallback Manager (Rescue Queries)
    if len(all_results) < 5 and any(s in ["alicia", "renati", "lareferencia"] for s in expanded):
        try:
            rescue = await PROVIDERS["openalex"](query, limit=10)
            if rescue:
                for r in rescue:
                    r["source"] = f"{r.get('source', 'OpenAlex')} (Rescue)"
                all_results.extend(rescue)
        except Exception as e:
            pass

    # Deduplicate
    deduplicated = deduplicate_results(all_results)

    # Sort by year (newest first) and limit
    deduplicated.sort(key=lambda x: x.get("year") or 0, reverse=True)
    results = deduplicated[:max_results]

    return {
        "results": results,
        "total": len(results),
        "totalBeforeDedup": len(all_results),
        "sourcesUsed": expanded,
    }