Update src/api/agent.py

src/api/agent.py

@@ -1410,27 +1410,24 @@ async def _semantic_scholar_search(topic: str, limit: int = 8) -> List[dict]:
 
 async def _openalex_search(topic: str, limit: int = 8) -> List[dict]:
     """
-    Search OpenAlex using the ?search= relevance engine.
-    This matches the clickable OpenAlex link shown to users.
-    The previous ?filter=title.search:X|abstract.search:X approach was a strict
-    field-level filter that returned irrelevant results. The ?search= parameter
-    uses OpenAlex full relevance ranking across all fields.
+    Search OpenAlex using semantic search — passes the corrected natural language
+    query directly, matching the search.semantic format the user sees in the
+    OpenAlex browser link:
+      https://openalex.org/works?search.semantic=QUERY&sort=relevance_score:desc
+    The API equivalent uses ?search=QUERY which triggers the same semantic ranking.
     """
     try:
         from urllib.parse import quote as _oa_quote
 
-        # Strip boolean operators and common retrieval filler words.
+        # Strip only boolean operators — keep the full natural language query intact
+        # including proper nouns, locations, and all keywords
         clean = re.sub(r'\b(AND|OR|NOT)\b|[()"\']', ' ', topic, flags=re.IGNORECASE)
-        clean = re.sub(
-            r'\b(find|search|show|get|give|research on|articles on'
-            r'|papers on|literature on|studies on|overview of|summary of|review of)\b',
-            ' ', clean, flags=re.IGNORECASE,
-        )
-        clean = re.sub(r'\s+', ' ', clean).strip()[:150]
+        clean = re.sub(r'\s+', ' ', clean).strip()[:200]
         encoded = _oa_quote(clean, safe='')
 
         select = "id,title,authorships,publication_year,primary_location,abstract_inverted_index,doi,cited_by_count,open_access"
 
+        # Use ?search= — API equivalent of search.semantic in the browser link
         url = (
             f"https://api.openalex.org/works"
             f"?search={encoded}"
@@ -2377,29 +2374,22 @@ def _highly_cited_note(topic_escaped: str) -> str:
 
 async def _research_snapshot(context: SearchContextPayload, model: str) -> Tuple[str, List[dict], List[dict]]:
     """
-    Research snapshot:
-    - Uses Consensus MCP for peer-reviewed papers with abstracts (primary).
-    - Falls back to Semantic Scholar silently if Consensus returns nothing.
-    - Runs PRIMO in parallel for KU-accessible results.
-    - Reasoning block at the top, cited summary in the middle, AI tools at the end.
+    Research snapshot using OpenAlex semantic search only.
+    PRIMO API removed — PRIMO boolean is used only for the browser link shown to users.
+    Falls back to Semantic Scholar if OpenAlex returns nothing.
     """
     import asyncio as _asyncio
     topic = context.topic
     references: List[dict] = []
 
-    # Parallel fetch: OpenAlex (fast, free) + PRIMO
-    papers, (primo_result, primo_url) = await _asyncio.gather(
-        _openalex_search(context.ai_tool_query or context.topic, limit=8),
-        _search_primo_catalog(context, limit=6),
-    )
+    # OpenAlex semantic search — passes corrected natural language query
+    papers = await _openalex_search(context.ai_tool_query or context.topic, limit=8)
 
     # Silent fallback to Semantic Scholar if OpenAlex returned nothing
     if not papers:
         logger.info(f"OpenAlex returned no papers for '{topic}' — trying Semantic Scholar")
         papers = await _semantic_scholar_search(topic, limit=8)
 
-    search_results = primo_result.get("results", []) or []
-
     # ── Top: reasoning block ──────────────────────────────────────────────────
     q_display = context.ai_tool_query or f"Research on {topic}"
     answer = _search_trace_block(q_display, context)
@@ -2412,24 +2402,18 @@ async def _research_snapshot(context: SearchContextPayload, model: str) -> Tuple
 
         source_label = papers[0].get("_source", "OpenAlex")
         if source_label == "OpenAlex":
-            source_url = "https://openalex.org"
+            oa_source_url = "https://openalex.org"
             source_key = "openalex_badge"
         elif source_label == "Consensus":
-            source_url = "https://consensus.app"
+            oa_source_url = "https://consensus.app"
             source_key = "consensus_badge"
         else:
-            source_url = "https://www.semanticscholar.org/"
+            oa_source_url = "https://www.semanticscholar.org/"
             source_key = "semantic"
 
         # ── Middle: cited summary ─────────────────────────────────────────────
         answer += f"<strong>📖 Research Snapshot: {_escape(topic)}</strong><br><br>{summary}"
 
-        if search_results:
-            answer += (
-                "<br><br><strong>KU Library results:</strong> "
-                "KU-accessible items are shown below."
-            )
-
         answer += "<br><br><strong>Selected references:</strong><br>"
         for i, paper in enumerate(papers[:6], 1):
             answer += (
@@ -2440,31 +2424,18 @@ async def _research_snapshot(context: SearchContextPayload, model: str) -> Tuple
                 f"</span>"
             )
 
-        # Source badge — show Consensus or Semantic Scholar depending on which fired
-        answer += _source_badge(source_key, source_url)
-        if search_results:
-            answer += _source_badge("primo", primo_url)
-
-        # ── Bottom: AI tools ──────────────────────────────────────────────────
+        answer += _source_badge(source_key, oa_source_url)
         answer += _ai_tools_footer(context)
         references = [{"title": p["title"], "url": p["link"]} for p in papers[:6]]
-        return answer, search_results, references
+        return answer, [], references
 
-    # Both Consensus and Semantic Scholar returned nothing — PRIMO-only fallback
-    if search_results:
-        answer += (
-            f"<strong>📚 Research starting points: {_escape(topic)}</strong><br><br>"
-            "Here are KU-accessible results you can open immediately."
-        )
-    else:
-        answer += (
-            f"<strong>📚 Research starting points: {_escape(topic)}</strong><br><br>"
-            "I didn't find direct results right now — try the AI research tools below."
-        )
-    answer += _source_badge("primo", primo_url)
+    # OpenAlex and Semantic Scholar both returned nothing
+    answer += (
+        f"<strong>📚 Research starting points: {_escape(topic)}</strong><br><br>"
+        "I didn't find direct results right now — try the AI research tools below."
+    )
     answer += _ai_tools_footer(context)
-    return answer, search_results, []
-
+    return answer, [], []
 
 # ── Follow-up System ──────────────────────────────────────────────────────────
 
@@ -2812,48 +2783,46 @@ async def _run_search_mode(
 ) -> Tuple[str, List[dict], List[dict], str]:
     context = await _prepare_queries(question, context, model, is_follow_up)
 
-    # Run PRIMO (or PubMed) and OpenAlex in parallel.
+    # OpenAlex semantic search only — PRIMO API removed.
+    # PRIMO boolean is preserved for the browser link shown to users.
     # Skip OpenAlex for books-only searches — it only indexes journal articles.
     use_openalex = context.resource_type != "books"
 
     import asyncio as _aio
     if context.source == "pubmed":
         if use_openalex:
-            (result, source_url), oa_papers = await _aio.gather(
+            (result, pub_source_url), oa_papers = await _aio.gather(
                 _search_pubmed_articles(context, limit=6),
                 _openalex_search(context.ai_tool_query or context.topic, limit=5),
             )
         else:
-            result, source_url = await _search_pubmed_articles(context, limit=6)
+            result, pub_source_url = await _search_pubmed_articles(context, limit=6)
             oa_papers = []
         source_name = "pubmed"
+        source_url = pub_source_url
+        search_results = result.get("results", []) or []
+        total = result.get("total", len(search_results))
+        for r in search_results:
+            if "_source" not in r:
+                r["_source"] = "PubMed"
     else:
+        # PRIMO API removed — use OpenAlex only for article results
         if use_openalex:
-            (result, source_url), oa_papers = await _aio.gather(
-                _search_primo_catalog(context, limit=6),
-                _openalex_search(context.ai_tool_query or context.topic, limit=5),
-            )
+            oa_papers = await _openalex_search(context.ai_tool_query or context.topic, limit=8)
         else:
-            result, source_url = await _search_primo_catalog(context, limit=6)
             oa_papers = []
-        source_name = "primo"
-
-    search_results = result.get("results", []) or []
-    total = result.get("total", len(search_results))
-
-    # Tag PRIMO/PubMed results with source
-    for r in search_results:
-        if "_source" not in r:
-            r["_source"] = "PubMed" if context.source == "pubmed" else "PRIMO"
+        source_name = "openalex"
+        source_url = _primo_clean_url(context)  # PRIMO link still shown to user
+        search_results = []
+        total = 0
 
-    # Tag OpenAlex results and add abstract as description
+    # Tag OpenAlex results
     for paper in oa_papers:
         paper["_source"] = "OpenAlex"
         paper["description"] = paper.get("abstract", "")[:300]
         paper["creator"] = paper.get("authors", "")
         paper["date"] = paper.get("year", "")
         paper["type"] = "Article"
-        # PRIMO search URL for "Read more" — searches by title in PRIMO
         from urllib.parse import quote as _pq
         primo_title_url = (
             "https://khalifa.primo.exlibrisgroup.com/discovery/search"
@@ -2866,42 +2835,47 @@ async def _run_search_mode(
     # ── Top: reasoning / how-I-searched block ────────────────────────────────
     answer = _search_trace_block(question, context)
 
-    # ── Middle: answer sentence + result count ────────────────────────────────
+    # ── Middle: answer sentence ────────────────────────────────────────────────
     answer += _search_answer_intro(context, is_follow_up)
 
     filters = _filters_summary(context)
     if filters:
         answer += f"<br><br><strong>Active filters:</strong> {_escape(filters)}."
 
-    if search_results:
-        answer += f"<br><br>Found <strong>{total:,}</strong> result(s) — top {len(search_results)} shown below."
+    if context.source == "pubmed":
+        if search_results:
+            answer += f"<br><br>Found <strong>{total:,}</strong> result(s) — top {len(search_results)} shown below."
+        else:
+            answer += "<br><br>No PubMed results found — try the AI tools below."
+        answer += _source_badge(source_name, source_url)
+        if oa_papers:
+            answer += _source_badge("openalex_badge", "https://openalex.org")
     else:
-        answer += "<br><br>No matching results found right now — try broadening your terms or use the AI tools below."
+        if oa_papers:
+            answer += f"<br><br>Found <strong>{len(oa_papers)}</strong> open access result(s) via OpenAlex."
+        else:
+            answer += "<br><br>No results found right now — try the AI tools or the PRIMO link below."
+        answer += _source_badge("openalex_badge", "https://openalex.org")
 
     if context.intent == "search_medical" and context.source != "pubmed":
         answer += "<br><br>For the strongest medical evidence trail, also check <strong>PubMed</strong> or <strong>Embase</strong>."
 
-    answer += _source_badge(source_name, source_url)
-
-    if oa_papers:
-        answer += _source_badge("openalex_badge", "https://openalex.org")
-
     # ── Bottom: AI tools recommendation panel ─────────────────────────────────
     answer += _ai_tools_footer(context)
 
-    # Merge results: PRIMO first, then OpenAlex — frontend splits by _source
-    combined_results = search_results + oa_papers[:5]
+    # Combined results: PubMed (if medical) + OpenAlex
+    combined_results = search_results + oa_papers[:8]
 
     sources = []
     for item in search_results[:6]:
         link = item.get("primo_url") or item.get("link")
         if link:
             sources.append({"title": item.get("title", "Result"), "url": link})
-    for paper in oa_papers[:3]:
+    for paper in oa_papers[:6]:
         if paper.get("link"):
             sources.append({"title": paper["title"], "url": paper["link"]})
-    return answer, combined_results, sources, source_url
 
+    return answer, combined_results, sources, source_url
 
 def _sort_and_trim_contexts(contexts: List[SearchContextPayload]) -> List[SearchContextPayload]:
     dedup: Dict[str, SearchContextPayload] = {}
@@ -3508,40 +3482,47 @@ async def _agent_query_inner(
                 _prepare_queries(question, _ctx_articles, model, False),
                 _prepare_queries(question, _ctx_books, model, False),
             )
-            (_res_a, _src_url_a), (_res_b, _src_url_b) = await _aio.gather(
-                _search_primo_catalog(_ctx_articles, limit=4),
-                _search_primo_catalog(_ctx_books, limit=3),
-            )
-            _results_a = _res_a.get("results", [])
-            _results_b = _res_b.get("results", [])
-            _total_a = _res_a.get("total", len(_results_a))
-            _total_b = _res_b.get("total", len(_results_b))
+            # PRIMO API removed — use OpenAlex for articles, PRIMO link for books
+            _ctx_articles_prepared = await _prepare_queries(question, _ctx_articles, model, False)
+            _oa_articles = await _openalex_search(_ctx_articles_prepared.ai_tool_query or topic, limit=6)
+            for _p in _oa_articles:
+                _p["_source"] = "OpenAlex"
+                _p["description"] = _p.get("abstract", "")[:300]
+                _p["creator"] = _p.get("authors", "")
+                _p["date"] = _p.get("year", "")
+                _p["type"] = "Article"
+            _results_a = _oa_articles
+            _results_b = []  # books: direct user to PRIMO link
+            _src_url_a = _primo_clean_url(_ctx_articles)
+            _src_url_b = _primo_clean_url(_ctx_books)
 
             _dual_answer = _search_trace_block(question, _ctx_articles)
             _dual_answer += (
-                f"<strong>📄 Articles on {_escape(topic)}</strong> "
-                f"— {_total_a:,} result(s) found, top {len(_results_a)} shown below.<br>"
+                f"<strong>📄 Articles on {_escape(topic)}</strong>"
+            )
+            if _results_a:
+                _dual_answer += f" — {len(_results_a)} open access result(s) shown below.<br>"
+            else:
+                _dual_answer += "<br><em>No article results found — try the PRIMO link below.</em><br>"
+            _dual_answer += _source_badge("openalex_badge", "https://openalex.org")
+
+            _dual_answer += f"<br><br><strong>📚 Books on {_escape(topic)}</strong><br>"
+            _dual_answer += (
+                "For books, search the KU Library catalogue directly: "
+                f'<a href="{_src_url_b}" target="_blank" style="color:#003366;font-weight:700">Search PRIMO for books →</a><br>'
             )
-            if not _results_a:
-                _dual_answer += "<em>No article results found — try broadening your search.</em><br>"
-            _dual_answer += _source_badge("primo", _src_url_a)
-
-            _dual_answer += f"<br><br><strong>📚 Books on {_escape(topic)}</strong> "
-            _dual_answer += f"— {_total_b:,} result(s) found, top {len(_results_b)} shown below.<br>"
-            if not _results_b:
-                _dual_answer += "<em>No book results found — try the KU Library catalogue directly.</em><br>"
             _dual_answer += _source_badge("primo", _src_url_b)
             _dual_answer += _ai_tools_footer(_ctx_articles)
 
-            _combined = _results_a + _results_b
+            _combined = _results_a
             follow_up_question, follow_up_suggestions = await _search_follow_up(_ctx_articles, model)
             return AgentResponse(
                 answer=_dual_answer,
                 intent=intent,
-                tools_used=["primo_dual_resource"],
+                tools_used=["openalex_articles", "primo_books_link"],
                 search_results=_combined[:8],
-                sources=[{"title": r.get("title", "Result"), "url": r.get("primo_url") or r.get("link", "")}
-                         for r in _combined[:6] if r.get("primo_url") or r.get("link")],
+                sources=[{"title": r.get("title", "Result"), "url": r.get("link", "")}
+                         for r in _combined[:6] if r.get("link")],
                 model_used=model,
                 response_time=time.time() - start_time,
                 corrected_query=topic,