Update src/api/agent.py

src/api/agent.py

@@ -416,6 +416,7 @@ def _source_badge(source: str, url: str = "") -> str:
         "primo": ("📚", "KU Library PRIMO", "#eff6ff", "#1e40af"),
         "semantic": ("🎓", "Semantic Scholar", "#f0fdf4", "#166534"),
         "consensus_badge": ("🧬", "Consensus", "#fdf4ff", "#86198f"),
+        "openalex_badge": ("🌍", "OpenAlex", "#f0fdf4", "#166534"),
         "pubmed": ("🏥", "PubMed", "#d1fae5", "#065f46"),
         "web": ("🌐", "Web Search", "#f5f3ff", "#4c1d95"),
         "web_live": ("🌐", "Web Search · Live", "#fdf4ff", "#86198f"),
@@ -1396,139 +1397,92 @@ async def _semantic_scholar_search(topic: str, limit: int = 8) -> List[dict]:
         return []
 
 
-async def _consensus_mcp_search(topic: str, model: str, limit: int = 8) -> List[dict]:
+async def _openalex_search(topic: str, limit: int = 8) -> List[dict]:
     """
-    Search Consensus via Anthropic MCP to get real peer-reviewed papers with abstracts.
-    Returns papers in the same dict format as _semantic_scholar_search so the rest of
-    _research_snapshot works unchanged.
-    Falls back silently to Semantic Scholar if Consensus MCP fails.
+    Search OpenAlex (free, no API key) for peer-reviewed papers with abstracts.
+    Returns papers in the same dict format as _semantic_scholar_search.
+    Typically responds in < 2 seconds.
     """
-    settings = get_settings()
-    if not settings.anthropic_api_key:
-        return []
-
-    prompt = (
-        f"Search for {limit} high-quality peer-reviewed papers on this topic: {topic}\n\n"
-        f"For each paper return: title, authors, year, abstract (2-3 sentences), "
-        f"journal/venue, and DOI or URL. Focus on influential and recent papers."
-    )
     try:
-        async with httpx.AsyncClient(timeout=30) as client:
-            r = await client.post(
-                "https://api.anthropic.com/v1/messages",
-                headers={
-                    "Content-Type": "application/json",
-                    "x-api-key": settings.anthropic_api_key,
-                    "anthropic-version": "2023-06-01",
-                    "anthropic-beta": "mcp-client-2025-04-04",
-                },
-                json={
-                    "model": "claude-haiku-4-5-20251001",
-                    "max_tokens": 2000,
-                    "messages": [{"role": "user", "content": prompt}],
-                    "mcp_servers": [{
-                        "type": "url",
-                        "url": "https://mcp.consensus.app/mcp",
-                        "name": "consensus"
-                    }],
-                },
-            )
-            if r.status_code != 200:
-                logger.warning(f"Consensus MCP returned {r.status_code}")
-                return []
-
-            data = r.json()
-            # Collect all text blocks (tool results + final text)
-            text_parts = []
-            for block in data.get("content", []):
-                if block.get("type") == "text":
-                    text_parts.append(block["text"])
-                elif block.get("type") == "mcp_tool_result":
-                    for item in block.get("content", []):
-                        if item.get("type") == "text":
-                            text_parts.append(item["text"])
-            full_text = "\n".join(text_parts)
-            if not full_text.strip():
-                return []
-
-            # Parse the structured text into paper dicts using LLM
-            papers = await _parse_consensus_response(full_text, topic, settings.anthropic_api_key)
-            logger.info(f"Consensus MCP returned {len(papers)} papers for: {topic[:50]}")
-            return papers[:limit]
-
-    except Exception as e:
-        logger.warning(f"Consensus MCP search failed: {e} — will fall back to Semantic Scholar")
-        return []
-
+        from urllib.parse import quote as _q
+        params = {
+            "search": topic,
+            "filter": "has_abstract:true,type:article",
+            "sort": "cited_by_count:desc",
+            "per_page": limit,
+            "select": "id,title,authorships,publication_year,primary_location,abstract_inverted_index,doi,cited_by_count,open_access",
+        }
+        url = "https://api.openalex.org/works?" + "&".join(f"{k}={_q(str(v))}" for k, v in params.items())
+        headers = {"User-Agent": "LibBee/3.2 (mailto:libse@ku.ac.ae)"}
 
-async def _parse_consensus_response(text: str, topic: str, api_key: str) -> List[dict]:
-    """Parse Consensus MCP response text into structured paper dicts."""
-    try:
-        async with httpx.AsyncClient(timeout=15) as client:
-            r = await client.post(
-                "https://api.anthropic.com/v1/messages",
-                headers={
-                    "Content-Type": "application/json",
-                    "x-api-key": api_key,
-                    "anthropic-version": "2023-06-01",
-                },
-                json={
-                    "model": "claude-haiku-4-5-20251001",
-                    "max_tokens": 1500,
-                    "messages": [{
-                        "role": "user",
-                        "content": (
-                            "Extract the research papers from this text into a JSON array. "
-                            "Each paper object must have: title, authors (string), year (string), "
-                            "abstract (string, keep it), venue (journal name), doi (string or null), url (string or null). "
-                            "Return ONLY valid JSON array, no markdown, no preamble.\n\n"
-                            f"Text:\n{text[:4000]}"
-                        )
-                    }],
-                },
-            )
+        async with httpx.AsyncClient(timeout=10) as client:
+            r = await client.get(url, headers=headers)
             if r.status_code != 200:
-                return []
-            raw = "".join(
-                b["text"] for b in r.json().get("content", []) if b.get("type") == "text"
-            ).strip()
-            # Strip markdown fences if present
-            if raw.startswith("```"):
-                raw = raw.split("\n", 1)[1].rsplit("```", 1)[0].strip()
-            papers_raw = json.loads(raw)
-            if not isinstance(papers_raw, list):
+                logger.warning(f"OpenAlex returned {r.status_code}")
                 return []
 
+            data = r.json()
             results = []
-            for p in papers_raw:
-                if not isinstance(p, dict) or not p.get("title"):
+            for work in data.get("results", []):
+                title = work.get("title") or "Untitled"
+
+                # Reconstruct abstract from inverted index
+                inv = work.get("abstract_inverted_index") or {}
+                abstract = ""
+                if inv:
+                    word_positions = []
+                    for word, positions in inv.items():
+                        for pos in positions:
+                            word_positions.append((pos, word))
+                    word_positions.sort(key=lambda x: x[0])
+                    abstract = " ".join(w for _, w in word_positions)[:700]
+
+                if not abstract:
                     continue
-                doi = (p.get("doi") or "").strip()
-                url = (p.get("url") or "").strip()
+
+                # Authors
+                authorships = work.get("authorships") or []
+                author_names = [
+                    a.get("author", {}).get("display_name", "")
+                    for a in authorships[:4]
+                    if a.get("author", {}).get("display_name")
+                ]
+                authors_str = ", ".join(author_names[:3])
+                if len(author_names) > 3:
+                    authors_str += " et al."
+                first_last = (author_names[0].split()[-1] if author_names else "Unknown")
+
+                year = str(work.get("publication_year") or "n.d.")
+                doi = (work.get("doi") or "").replace("https://doi.org/", "")
+                oa_url = (work.get("open_access") or {}).get("oa_url") or ""
                 link = (
-                    url if url and url.startswith("http")
+                    oa_url if oa_url
                     else (f"https://doi.org/{doi}" if doi else "")
-                    or f"https://consensus.app/results/?q={quote(p.get('title', ''))}"
+                    or work.get("id", "https://openalex.org")
                 )
-                authors_raw = p.get("authors", "Unknown")
-                author_list = [a.strip() for a in str(authors_raw).split(",")]
-                first_last = author_list[0].split()[-1] if author_list else "Unknown"
-                year = str(p.get("year", "n.d.") or "n.d.")
+                venue = ((work.get("primary_location") or {}).get("source") or {}).get("display_name") or ""
+
                 results.append({
-                    "title": p.get("title", "Untitled"),
-                    "authors": ", ".join(author_list[:3]) + (" et al." if len(author_list) > 3 else ""),
+                    "title": title,
+                    "authors": authors_str or "Unknown",
                     "first_author": first_last,
                     "year": year,
                     "citation_key": f"{first_last}, {year}",
-                    "venue": p.get("venue", "") or "",
-                    "abstract": (p.get("abstract") or "")[:700],
+                    "venue": venue,
+                    "abstract": abstract,
                     "link": link,
-                    "_source": "Consensus",
+                    "_source": "OpenAlex",
                 })
+
+            logger.info(f"OpenAlex: {len(results)} papers for '{topic[:50]}'")
             return results
+
     except Exception as e:
-        logger.warning(f"_parse_consensus_response failed: {e}")
+        logger.warning(f"OpenAlex search failed: {e}")
         return []
+
+
+async def _generate_cited_summary(topic: str, papers: List[dict], model: str) -> str:
     settings = get_settings()
     if not settings.openai_api_key and not settings.anthropic_api_key:
         return ""
@@ -2401,15 +2355,15 @@ async def _research_snapshot(context: SearchContextPayload, model: str) -> Tuple
     topic = context.topic
     references: List[dict] = []
 
-    # Parallel fetch: Consensus MCP + PRIMO
+    # Parallel fetch: OpenAlex (fast, free) + PRIMO
     papers, (primo_result, primo_url) = await _asyncio.gather(
-        _consensus_mcp_search(topic, model, limit=8),
+        _openalex_search(topic, limit=8),
         _search_primo_catalog(context, limit=6),
     )
 
-    # Silent fallback to Semantic Scholar if Consensus returned nothing
+    # Silent fallback to Semantic Scholar if OpenAlex returned nothing
     if not papers:
-        logger.info(f"Consensus returned no papers for '{topic}' — trying Semantic Scholar")
+        logger.info(f"OpenAlex returned no papers for '{topic}' — trying Semantic Scholar")
         papers = await _semantic_scholar_search(topic, limit=8)
 
     search_results = primo_result.get("results", []) or []
@@ -2424,9 +2378,16 @@ async def _research_snapshot(context: SearchContextPayload, model: str) -> Tuple
             _safe_metrics_increment("summary_fallbacks")
             summary = _extractive_snapshot_from_papers(topic, papers)
 
-        source_label = papers[0].get("_source", "Consensus")
-        source_url = "https://consensus.app" if source_label == "Consensus" else "https://www.semanticscholar.org/"
-        source_key = "consensus_badge" if source_label == "Consensus" else "semantic"
+        source_label = papers[0].get("_source", "OpenAlex")
+        if source_label == "OpenAlex":
+            source_url = "https://openalex.org"
+            source_key = "openalex_badge"
+        elif source_label == "Consensus":
+            source_url = "https://consensus.app"
+            source_key = "consensus_badge"
+        else:
+            source_url = "https://www.semanticscholar.org/"
+            source_key = "semantic"
 
         # ── Middle: cited summary ─────────────────────────────────────────────
         answer += f"<strong>📖 Research Snapshot: {_escape(topic)}</strong><br><br>{summary}"