VibecoderMcSwaggins commited on
Commit
fd28242
·
1 Parent(s): 1cd600b

feat(tools): add OpenAlex integration with citation-based ranking

Browse files

- Implemented OpenAlexTool in src/tools/openalex.py
- Added comprehensive unit and integration tests in tests/unit/tools/test_openalex.py
- Updated src/tools/__init__.py to export the new tool
- Updated SPEC_03 with final implementation details and test patterns
- Features: Citation count sorting, abstract reconstruction, concept tagging, OA links
- Verified with 100% test pass rate and clean lint/type checks

docs/specs/SPEC_03_OPENALEX_INTEGRATION.md CHANGED
@@ -4,16 +4,17 @@
4
 
5
  ## Problem Statement
6
 
7
- We search 3 sources (PubMed, Europe PMC, ClinicalTrials.gov) but have **no citation metrics**. We can't distinguish a highly-cited landmark paper from an obscure one. OpenAlex provides:
8
 
9
  1. **Citation counts** - Prioritize authoritative papers
10
  2. **Citation networks** - "Who cites whom"
11
  3. **Concept tagging** - Hierarchical categorization
12
- 4. **Related works** - ML-powered similarity
13
- 5. **Open access links** - Direct PDF URLs
14
 
15
  **FREE API. No key required. 209M+ works indexed.**
16
 
 
 
17
  ## Groundwork Already Done
18
 
19
  ```python
@@ -42,7 +43,7 @@ GET https://api.openalex.org/works
42
  | Parameter | Description |
43
  |-----------|-------------|
44
  | `search` | Full-text search across title, abstract, fulltext |
45
- | `filter` | Constrain results (e.g., `type:article`) |
46
  | `sort` | Order results (e.g., `cited_by_count:desc`) |
47
  | `per_page` | Results per page (max 200) |
48
  | `mailto` | Email for polite pool (higher rate limits) |
@@ -50,7 +51,7 @@ GET https://api.openalex.org/works
50
  ### Example Request
51
 
52
  ```bash
53
- GET https://api.openalex.org/works?search=metformin%20cancer&filter=type:article&sort=cited_by_count:desc&per_page=10&mailto=deepboner@example.com
54
  ```
55
 
56
  ### Response Structure
@@ -86,28 +87,6 @@ GET https://api.openalex.org/works?search=metformin%20cancer&filter=type:article
86
  }
87
  ```
88
 
89
- ### Abstract Reconstruction
90
-
91
- OpenAlex stores abstracts as inverted index for compression. Must reconstruct:
92
-
93
- ```python
94
- def _reconstruct_abstract(self, inverted_index: dict[str, list[int]] | None) -> str:
95
- """Rebuild abstract from {"word": [positions]} format."""
96
- if not inverted_index:
97
- return ""
98
-
99
- position_word: dict[int, str] = {}
100
- for word, positions in inverted_index.items():
101
- for pos in positions:
102
- position_word[pos] = word
103
-
104
- if not position_word:
105
- return ""
106
-
107
- max_pos = max(position_word.keys())
108
- return " ".join(position_word.get(i, "") for i in range(max_pos + 1))
109
- ```
110
-
111
  ## Architecture
112
 
113
  ### Class Diagram
@@ -124,7 +103,7 @@ def _reconstruct_abstract(self, inverted_index: dict[str, list[int]] | None) ->
124
  │ OpenAlexTool │
125
  │ ───────────────────────────────── │
126
  │ - BASE_URL: str │
127
- │ - _polite_email: str
128
  │ ───────────────────────────────── │
129
  │ + name → "openalex" │
130
  │ + search(query, max_results) → list[Evidence] │
@@ -135,29 +114,6 @@ def _reconstruct_abstract(self, inverted_index: dict[str, list[int]] | None) ->
135
  └─────────────────────────────────────┘
136
  ```
137
 
138
- ### Integration Flow
139
-
140
- ```
141
- SearchHandler
142
-
143
- ├── PubMedTool
144
- ├── EuropePMCTool
145
- ├── ClinicalTrialsTool
146
- └── OpenAlexTool ◄── NEW
147
-
148
-
149
- Evidence(
150
- content=abstract[:2000],
151
- citation=Citation(source="openalex", ...),
152
- metadata={
153
- "cited_by_count": 150,
154
- "concepts": ["Metformin", "Cancer"],
155
- "is_open_access": True,
156
- "pdf_url": "https://..."
157
- }
158
- )
159
- ```
160
-
161
  ## TDD Implementation Plan
162
 
163
  ### Red Phase: Write Failing Tests First
@@ -167,7 +123,7 @@ SearchHandler
167
  ```python
168
  """Unit tests for OpenAlex tool - TDD RED phase."""
169
 
170
- from unittest.mock import AsyncMock, MagicMock, patch
171
 
172
  import pytest
173
 
@@ -215,77 +171,61 @@ class TestOpenAlexTool:
215
  def tool(self) -> OpenAlexTool:
216
  return OpenAlexTool()
217
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
218
  def test_tool_name(self, tool: OpenAlexTool) -> None:
219
  """Tool name should be 'openalex'."""
220
  assert tool.name == "openalex"
221
 
222
  @pytest.mark.asyncio
223
- async def test_search_returns_evidence(self, tool: OpenAlexTool) -> None:
224
  """Search should return Evidence objects."""
225
- with patch("httpx.AsyncClient") as mock_client:
226
- mock_instance = AsyncMock()
227
- mock_client.return_value.__aenter__.return_value = mock_instance
228
 
229
- mock_resp = MagicMock()
230
- mock_resp.json.return_value = SAMPLE_OPENALEX_RESPONSE
231
- mock_resp.raise_for_status.return_value = None
232
- mock_instance.get.return_value = mock_resp
233
-
234
- results = await tool.search("metformin cancer", max_results=5)
235
-
236
- assert len(results) == 1
237
- assert isinstance(results[0], Evidence)
238
- assert results[0].citation.source == "openalex"
239
 
240
  @pytest.mark.asyncio
241
- async def test_search_includes_citation_count(self, tool: OpenAlexTool) -> None:
242
  """Evidence metadata should include cited_by_count."""
243
- with patch("httpx.AsyncClient") as mock_client:
244
- mock_instance = AsyncMock()
245
- mock_client.return_value.__aenter__.return_value = mock_instance
246
-
247
- mock_resp = MagicMock()
248
- mock_resp.json.return_value = SAMPLE_OPENALEX_RESPONSE
249
- mock_resp.raise_for_status.return_value = None
250
- mock_instance.get.return_value = mock_resp
251
 
252
- results = await tool.search("metformin cancer", max_results=5)
253
-
254
- assert results[0].metadata["cited_by_count"] == 150
 
 
 
255
 
256
  @pytest.mark.asyncio
257
- async def test_search_includes_concepts(self, tool: OpenAlexTool) -> None:
258
  """Evidence metadata should include concepts."""
259
- with patch("httpx.AsyncClient") as mock_client:
260
- mock_instance = AsyncMock()
261
- mock_client.return_value.__aenter__.return_value = mock_instance
262
-
263
- mock_resp = MagicMock()
264
- mock_resp.json.return_value = SAMPLE_OPENALEX_RESPONSE
265
- mock_resp.raise_for_status.return_value = None
266
- mock_instance.get.return_value = mock_resp
267
-
268
- results = await tool.search("metformin cancer", max_results=5)
269
-
270
- assert "Metformin" in results[0].metadata["concepts"]
271
- assert "Cancer" in results[0].metadata["concepts"]
272
-
273
  @pytest.mark.asyncio
274
- async def test_search_includes_open_access_info(self, tool: OpenAlexTool) -> None:
275
  """Evidence metadata should include open access info."""
276
- with patch("httpx.AsyncClient") as mock_client:
277
- mock_instance = AsyncMock()
278
- mock_client.return_value.__aenter__.return_value = mock_instance
279
-
280
- mock_resp = MagicMock()
281
- mock_resp.json.return_value = SAMPLE_OPENALEX_RESPONSE
282
- mock_resp.raise_for_status.return_value = None
283
- mock_instance.get.return_value = mock_resp
284
-
285
- results = await tool.search("metformin cancer", max_results=5)
286
-
287
- assert results[0].metadata["is_open_access"] is True
288
- assert results[0].metadata["pdf_url"] == "https://example.com/paper.pdf"
289
 
290
  def test_reconstruct_abstract(self, tool: OpenAlexTool) -> None:
291
  """Abstract reconstruction from inverted index."""
@@ -297,68 +237,37 @@ class TestOpenAlexTool:
297
  "a": [4],
298
  "test": [5],
299
  }
300
-
301
  result = tool._reconstruct_abstract(inverted_index)
302
-
303
  assert result == "Hello world this is a test"
304
 
305
  def test_reconstruct_abstract_empty(self, tool: OpenAlexTool) -> None:
306
- """Handle None/empty inverted index."""
307
  assert tool._reconstruct_abstract(None) == ""
308
  assert tool._reconstruct_abstract({}) == ""
309
 
310
  @pytest.mark.asyncio
311
- async def test_search_empty_results(self, tool: OpenAlexTool) -> None:
312
  """Handle empty results gracefully."""
313
- with patch("httpx.AsyncClient") as mock_client:
314
- mock_instance = AsyncMock()
315
- mock_client.return_value.__aenter__.return_value = mock_instance
316
-
317
- mock_resp = MagicMock()
318
- mock_resp.json.return_value = {"results": []}
319
- mock_resp.raise_for_status.return_value = None
320
- mock_instance.get.return_value = mock_resp
321
-
322
- results = await tool.search("xyznonexistent123", max_results=5)
323
 
324
- assert results == []
325
 
326
- @pytest.mark.asyncio
327
- async def test_search_sorts_by_citations(self, tool: OpenAlexTool) -> None:
328
- """Verify API call requests citation-sorted results."""
329
- with patch("httpx.AsyncClient") as mock_client:
330
- mock_instance = AsyncMock()
331
- mock_client.return_value.__aenter__.return_value = mock_instance
332
-
333
- mock_resp = MagicMock()
334
- mock_resp.json.return_value = {"results": []}
335
- mock_resp.raise_for_status.return_value = None
336
- mock_instance.get.return_value = mock_resp
337
-
338
- await tool.search("test query", max_results=5)
339
-
340
- # Verify call params
341
- call_args = mock_instance.get.call_args
342
- params = call_args[1]["params"]
343
- assert params["sort"] == "cited_by_count:desc"
344
 
345
  @pytest.mark.asyncio
346
- async def test_search_uses_polite_pool(self, tool: OpenAlexTool) -> None:
347
- """Verify mailto parameter for polite pool."""
348
- with patch("httpx.AsyncClient") as mock_client:
349
- mock_instance = AsyncMock()
350
- mock_client.return_value.__aenter__.return_value = mock_instance
351
-
352
- mock_resp = MagicMock()
353
- mock_resp.json.return_value = {"results": []}
354
- mock_resp.raise_for_status.return_value = None
355
- mock_instance.get.return_value = mock_resp
356
-
357
- await tool.search("test query", max_results=5)
358
-
359
- call_args = mock_instance.get.call_args
360
- params = call_args[1]["params"]
361
- assert "mailto" in params
362
  ```
363
 
364
  ### Green Phase: Implement to Pass Tests
@@ -416,7 +325,7 @@ class OpenAlexTool:
416
  """
417
  params: dict[str, str | int] = {
418
  "search": query,
419
- "filter": "type:article", # Only peer-reviewed articles
420
  "sort": "cited_by_count:desc", # Most cited first
421
  "per_page": min(max_results, 100),
422
  "mailto": self.POLITE_EMAIL,
@@ -448,6 +357,7 @@ class OpenAlexTool:
448
  # Reconstruct abstract from inverted index
449
  abstract = self._reconstruct_abstract(work.get("abstract_inverted_index"))
450
  if not abstract:
 
451
  abstract = f"[No abstract available. Cited by {cited_by_count} works.]"
452
 
453
  # Extract authors (limit to 5)
@@ -475,6 +385,10 @@ class OpenAlexTool:
475
  citation_badge = f"[Cited by {cited_by_count}] " if cited_by_count > 0 else ""
476
  content = f"{citation_badge}{abstract[:1900]}"
477
 
 
 
 
 
478
  return Evidence(
479
  content=content[:2000],
480
  citation=Citation(
@@ -484,7 +398,7 @@ class OpenAlexTool:
484
  date=str(year),
485
  authors=authors,
486
  ),
487
- relevance=min(1.0, cited_by_count / 1000), # Normalize to 0-1
488
  metadata={
489
  "cited_by_count": cited_by_count,
490
  "concepts": concepts,
@@ -556,13 +470,13 @@ __all__ = [
556
  | `test_tool_name` | Returns "openalex" | P0 |
557
  | `test_search_returns_evidence` | Returns `list[Evidence]` | P0 |
558
  | `test_search_includes_citation_count` | `metadata["cited_by_count"]` populated | P0 |
 
559
  | `test_search_includes_concepts` | `metadata["concepts"]` populated | P0 |
560
- | `test_search_includes_open_access_info` | `metadata["is_open_access"]`, `pdf_url` | P1 |
561
  | `test_reconstruct_abstract` | Inverted index → text | P0 |
562
- | `test_reconstruct_abstract_empty` | Handle None/empty | P0 |
563
  | `test_search_empty_results` | Return `[]` for no matches | P0 |
564
- | `test_search_sorts_by_citations` | API param `sort=cited_by_count:desc` | P0 |
565
- | `test_search_uses_polite_pool` | API param `mailto` present | P1 |
566
 
567
  ## Integration Test
568
 
@@ -580,59 +494,8 @@ class TestOpenAlexIntegration:
580
  assert len(results) > 0
581
  # Should have citation counts
582
  assert results[0].metadata["cited_by_count"] >= 0
 
 
583
  # Should have concepts
584
  assert len(results[0].metadata["concepts"]) > 0
585
  ```
586
-
587
- ## Acceptance Criteria
588
-
589
- - [ ] `OpenAlexTool` implements `SearchTool` Protocol
590
- - [ ] `search()` returns `list[Evidence]` sorted by citation count
591
- - [ ] `metadata["cited_by_count"]` populated for each result
592
- - [ ] `metadata["concepts"]` contains top 5 concept names
593
- - [ ] `metadata["is_open_access"]` and `pdf_url` populated
594
- - [ ] Abstract correctly reconstructed from inverted index
595
- - [ ] Empty results handled gracefully
596
- - [ ] Tool exported from `src/tools/__init__.py`
597
- - [ ] All unit tests pass with mocked API
598
- - [ ] Integration test passes with real API
599
- - [ ] `make check` passes (lint + type + test)
600
-
601
- ## Files to Create/Modify
602
-
603
- ### Create
604
-
605
- 1. `src/tools/openalex.py` - Main tool class
606
- 2. `tests/unit/tools/test_openalex.py` - Unit tests
607
-
608
- ### Modify
609
-
610
- 1. `src/tools/__init__.py` - Add export
611
- 2. `src/orchestrator.py` - Add to tool list (if using default tools)
612
-
613
- ## SOLID Principles Applied
614
-
615
- | Principle | Application |
616
- |-----------|------------|
617
- | **S**ingle Responsibility | `OpenAlexTool` only handles OpenAlex API |
618
- | **O**pen/Closed | New tool without modifying existing ones |
619
- | **L**iskov Substitution | Implements `SearchTool` Protocol, interchangeable |
620
- | **I**nterface Segregation | Uses minimal `SearchTool` interface |
621
- | **D**ependency Inversion | `SearchHandler` depends on Protocol, not concrete class |
622
-
623
- ## DRY Applied
624
-
625
- - Reuses `Evidence`, `Citation`, `SearchError` from `src/utils/models.py`
626
- - Follows identical patterns as `PubMedTool`, `EuropePMCTool`
627
- - Same retry decorator pattern as other tools
628
-
629
- ## Related Issues
630
-
631
- - #51: feat: Add OpenAlex as 4th data source
632
-
633
- ## Effort Estimate
634
-
635
- - Tests (TDD Red): 30 min
636
- - Implementation (Green): 1 hour
637
- - Refactor + Integration: 30 min
638
- - **Total: ~2 hours**
 
4
 
5
  ## Problem Statement
6
 
7
+ We currently search 3 sources (PubMed, Europe PMC, ClinicalTrials.gov) but lack **citation metrics**. We cannot distinguish a highly-cited landmark paper from an obscure one. OpenAlex provides:
8
 
9
  1. **Citation counts** - Prioritize authoritative papers
10
  2. **Citation networks** - "Who cites whom"
11
  3. **Concept tagging** - Hierarchical categorization
12
+ 4. **Open access links** - Direct PDF URLs
 
13
 
14
  **FREE API. No key required. 209M+ works indexed.**
15
 
16
+ > **Note:** This spec supersedes `docs/future-roadmap/phases/15_PHASE_OPENALEX.md`.
17
+
18
  ## Groundwork Already Done
19
 
20
  ```python
 
43
  | Parameter | Description |
44
  |-----------|-------------|
45
  | `search` | Full-text search across title, abstract, fulltext |
46
+ | `filter` | Constrain results (e.g., `type:article`, `has_abstract:true`) |
47
  | `sort` | Order results (e.g., `cited_by_count:desc`) |
48
  | `per_page` | Results per page (max 200) |
49
  | `mailto` | Email for polite pool (higher rate limits) |
 
51
  ### Example Request
52
 
53
  ```bash
54
+ GET https://api.openalex.org/works?search=metformin%20cancer&filter=type:article,has_abstract:true&sort=cited_by_count:desc&per_page=10&mailto=deepboner-research@proton.me
55
  ```
56
 
57
  ### Response Structure
 
87
  }
88
  ```
89
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
90
  ## Architecture
91
 
92
  ### Class Diagram
 
103
  │ OpenAlexTool │
104
  │ ───────────────────────────────── │
105
  │ - BASE_URL: str │
106
+ │ - POLITE_EMAIL: str
107
  │ ───────────────────────────────── │
108
  │ + name → "openalex" │
109
  │ + search(query, max_results) → list[Evidence] │
 
114
  └─────────────────────────────────────┘
115
  ```
116
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
117
  ## TDD Implementation Plan
118
 
119
  ### Red Phase: Write Failing Tests First
 
123
  ```python
124
  """Unit tests for OpenAlex tool - TDD RED phase."""
125
 
126
+ from unittest.mock import AsyncMock, MagicMock
127
 
128
  import pytest
129
 
 
171
  def tool(self) -> OpenAlexTool:
172
  return OpenAlexTool()
173
 
174
+ @pytest.fixture
175
+ def mock_client(self, mocker):
176
+ """Create a standardized mock client with context manager support."""
177
+ client = AsyncMock()
178
+ client.__aenter__.return_value = client
179
+ client.__aexit__.return_value = None
180
+
181
+ # Standard response mock
182
+ resp = MagicMock()
183
+ resp.json.return_value = SAMPLE_OPENALEX_RESPONSE
184
+ resp.raise_for_status.return_value = None
185
+ client.get.return_value = resp
186
+
187
+ mocker.patch("httpx.AsyncClient", return_value=client)
188
+ return client
189
+
190
  def test_tool_name(self, tool: OpenAlexTool) -> None:
191
  """Tool name should be 'openalex'."""
192
  assert tool.name == "openalex"
193
 
194
  @pytest.mark.asyncio
195
+ async def test_search_returns_evidence(self, tool: OpenAlexTool, mock_client) -> None:
196
  """Search should return Evidence objects."""
197
+ results = await tool.search("metformin cancer", max_results=5)
 
 
198
 
199
+ assert len(results) == 1
200
+ assert isinstance(results[0], Evidence)
201
+ assert results[0].citation.source == "openalex"
 
 
 
 
 
 
 
202
 
203
  @pytest.mark.asyncio
204
+ async def test_search_includes_citation_count(self, tool: OpenAlexTool, mock_client) -> None:
205
  """Evidence metadata should include cited_by_count."""
206
+ results = await tool.search("metformin cancer", max_results=5)
207
+ assert results[0].metadata["cited_by_count"] == 150
 
 
 
 
 
 
208
 
209
+ @pytest.mark.asyncio
210
+ async def test_search_calculates_relevance(self, tool: OpenAlexTool, mock_client) -> None:
211
+ """Evidence relevance should be based on citations (capped at 1.0)."""
212
+ results = await tool.search("metformin cancer", max_results=5)
213
+ # 150 citations / 100 = 1.5 -> capped at 1.0
214
+ assert results[0].relevance == 1.0
215
 
216
  @pytest.mark.asyncio
217
+ async def test_search_includes_concepts(self, tool: OpenAlexTool, mock_client) -> None:
218
  """Evidence metadata should include concepts."""
219
+ results = await tool.search("metformin cancer", max_results=5)
220
+ assert "Metformin" in results[0].metadata["concepts"]
221
+ assert "Cancer" in results[0].metadata["concepts"]
222
+
 
 
 
 
 
 
 
 
 
 
223
  @pytest.mark.asyncio
224
+ async def test_search_includes_open_access_info(self, tool: OpenAlexTool, mock_client) -> None:
225
  """Evidence metadata should include open access info."""
226
+ results = await tool.search("metformin cancer", max_results=5)
227
+ assert results[0].metadata["is_open_access"] is True
228
+ assert results[0].metadata["pdf_url"] == "https://example.com/paper.pdf"
 
 
 
 
 
 
 
 
 
 
229
 
230
  def test_reconstruct_abstract(self, tool: OpenAlexTool) -> None:
231
  """Abstract reconstruction from inverted index."""
 
237
  "a": [4],
238
  "test": [5],
239
  }
 
240
  result = tool._reconstruct_abstract(inverted_index)
 
241
  assert result == "Hello world this is a test"
242
 
243
  def test_reconstruct_abstract_empty(self, tool: OpenAlexTool) -> None:
244
+ """Handle None or empty inverted index."""
245
  assert tool._reconstruct_abstract(None) == ""
246
  assert tool._reconstruct_abstract({}) == ""
247
 
248
  @pytest.mark.asyncio
249
+ async def test_search_empty_results(self, tool: OpenAlexTool, mock_client) -> None:
250
  """Handle empty results gracefully."""
251
+ mock_client.get.return_value.json.return_value = {"results": []}
 
 
 
 
 
 
 
 
 
252
 
253
+ results = await tool.search("xyznonexistent123", max_results=5)
254
 
255
+ assert results == []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
256
 
257
  @pytest.mark.asyncio
258
+ async def test_search_params(self, tool: OpenAlexTool, mock_client) -> None:
259
+ """Verify API call requests citation-sorted results and uses polite pool."""
260
+ mock_client.get.return_value.json.return_value = {"results": []}
261
+
262
+ await tool.search("test query", max_results=5)
263
+
264
+ # Verify call params
265
+ call_args = mock_client.get.call_args
266
+ params = call_args[1]["params"]
267
+ assert params["sort"] == "cited_by_count:desc"
268
+ assert params["mailto"] == tool.POLITE_EMAIL
269
+ assert "type:article" in params["filter"]
270
+ assert "has_abstract:true" in params["filter"]
 
 
 
271
  ```
272
 
273
  ### Green Phase: Implement to Pass Tests
 
325
  """
326
  params: dict[str, str | int] = {
327
  "search": query,
328
+ "filter": "type:article,has_abstract:true", # Only articles with abstracts
329
  "sort": "cited_by_count:desc", # Most cited first
330
  "per_page": min(max_results, 100),
331
  "mailto": self.POLITE_EMAIL,
 
357
  # Reconstruct abstract from inverted index
358
  abstract = self._reconstruct_abstract(work.get("abstract_inverted_index"))
359
  if not abstract:
360
+ # Should be caught by filter=has_abstract:true, but defensive coding
361
  abstract = f"[No abstract available. Cited by {cited_by_count} works.]"
362
 
363
  # Extract authors (limit to 5)
 
385
  citation_badge = f"[Cited by {cited_by_count}] " if cited_by_count > 0 else ""
386
  content = f"{citation_badge}{abstract[:1900]}"
387
 
388
+ # Calculate relevance: normalized citation count (capped at 1.0 for 100 citations)
389
+ # 100 citations is a very strong signal in most fields.
390
+ relevance = min(1.0, cited_by_count / 100.0)
391
+
392
  return Evidence(
393
  content=content[:2000],
394
  citation=Citation(
 
398
  date=str(year),
399
  authors=authors,
400
  ),
401
+ relevance=relevance,
402
  metadata={
403
  "cited_by_count": cited_by_count,
404
  "concepts": concepts,
 
470
  | `test_tool_name` | Returns "openalex" | P0 |
471
  | `test_search_returns_evidence` | Returns `list[Evidence]` | P0 |
472
  | `test_search_includes_citation_count` | `metadata["cited_by_count"]` populated | P0 |
473
+ | `test_search_calculates_relevance` | `relevance` derived from citations | P1 |
474
  | `test_search_includes_concepts` | `metadata["concepts"]` populated | P0 |
475
+ | `test_search_includes_open_access_info` | `metadata["is_open_access"]` and `pdf_url` | P1 |
476
  | `test_reconstruct_abstract` | Inverted index → text | P0 |
477
+ | `test_reconstruct_abstract_empty` | Handle None/empty inputs | P1 |
478
  | `test_search_empty_results` | Return `[]` for no matches | P0 |
479
+ | `test_search_params` | API params (`sort`, `filter`, `mailto`) | P1 |
 
480
 
481
  ## Integration Test
482
 
 
494
  assert len(results) > 0
495
  # Should have citation counts
496
  assert results[0].metadata["cited_by_count"] >= 0
497
+ # Should have abstract text
498
+ assert len(results[0].content) > 50
499
  # Should have concepts
500
  assert len(results[0].metadata["concepts"]) > 0
501
  ```
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
src/tools/__init__.py CHANGED
@@ -3,6 +3,7 @@
3
  from src.tools.base import SearchTool
4
  from src.tools.clinicaltrials import ClinicalTrialsTool
5
  from src.tools.europepmc import EuropePMCTool
 
6
  from src.tools.pubmed import PubMedTool
7
  from src.tools.search_handler import SearchHandler
8
 
@@ -10,6 +11,7 @@ from src.tools.search_handler import SearchHandler
10
  __all__ = [
11
  "ClinicalTrialsTool",
12
  "EuropePMCTool",
 
13
  "PubMedTool",
14
  "SearchHandler",
15
  "SearchTool",
 
3
  from src.tools.base import SearchTool
4
  from src.tools.clinicaltrials import ClinicalTrialsTool
5
  from src.tools.europepmc import EuropePMCTool
6
+ from src.tools.openalex import OpenAlexTool
7
  from src.tools.pubmed import PubMedTool
8
  from src.tools.search_handler import SearchHandler
9
 
 
11
  __all__ = [
12
  "ClinicalTrialsTool",
13
  "EuropePMCTool",
14
+ "OpenAlexTool",
15
  "PubMedTool",
16
  "SearchHandler",
17
  "SearchTool",
src/tools/openalex.py ADDED
@@ -0,0 +1,162 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """OpenAlex search tool - citation-aware scholarly search."""
2
+
3
+ from typing import Any
4
+
5
+ import httpx
6
+ from tenacity import retry, stop_after_attempt, wait_exponential
7
+
8
+ from src.utils.exceptions import SearchError
9
+ from src.utils.models import Citation, Evidence
10
+
11
+
12
+ class OpenAlexTool:
13
+ """
14
+ Search OpenAlex for scholarly works with citation metrics.
15
+
16
+ OpenAlex indexes 209M+ works and provides:
17
+ - Citation counts (prioritize influential papers)
18
+ - Concept tagging (hierarchical classification)
19
+ - Open access links (direct PDF URLs)
20
+ - Related works (ML-powered similarity)
21
+
22
+ API Docs: https://docs.openalex.org
23
+ Rate Limits: Polite pool with mailto = 100k/day
24
+ """
25
+
26
+ BASE_URL = "https://api.openalex.org/works"
27
+ POLITE_EMAIL = "deepboner-research@proton.me"
28
+
29
+ @property
30
+ def name(self) -> str:
31
+ return "openalex"
32
+
33
+ @retry(
34
+ stop=stop_after_attempt(3),
35
+ wait=wait_exponential(multiplier=1, min=1, max=10),
36
+ reraise=True,
37
+ )
38
+ async def search(self, query: str, max_results: int = 10) -> list[Evidence]:
39
+ """
40
+ Search OpenAlex, sorted by citation count.
41
+
42
+ Args:
43
+ query: Search terms
44
+ max_results: Maximum results to return
45
+
46
+ Returns:
47
+ List of Evidence objects with citation metadata
48
+ """
49
+ params: dict[str, str | int] = {
50
+ "search": query,
51
+ "filter": "type:article,has_abstract:true", # Only articles with abstracts
52
+ "sort": "cited_by_count:desc", # Most cited first
53
+ "per_page": min(max_results, 100),
54
+ "mailto": self.POLITE_EMAIL,
55
+ }
56
+
57
+ async with httpx.AsyncClient(timeout=30.0) as client:
58
+ try:
59
+ response = await client.get(self.BASE_URL, params=params)
60
+ response.raise_for_status()
61
+
62
+ data = response.json()
63
+ works = data.get("results", [])
64
+
65
+ return [self._to_evidence(work) for work in works[:max_results]]
66
+
67
+ except httpx.HTTPStatusError as e:
68
+ raise SearchError(f"OpenAlex API error: {e}") from e
69
+ except httpx.RequestError as e:
70
+ raise SearchError(f"OpenAlex connection failed: {e}") from e
71
+
72
+ def _to_evidence(self, work: dict[str, Any]) -> Evidence:
73
+ """Convert OpenAlex work to Evidence with rich metadata."""
74
+ # Extract basic fields
75
+ title = work.get("display_name", "Untitled")
76
+ doi = work.get("doi", "")
77
+ year = work.get("publication_year", "Unknown")
78
+ cited_by_count = work.get("cited_by_count", 0)
79
+
80
+ # Reconstruct abstract from inverted index
81
+ abstract = self._reconstruct_abstract(work.get("abstract_inverted_index"))
82
+ if not abstract:
83
+ # Should be caught by filter=has_abstract:true, but defensive coding
84
+ abstract = f"[No abstract available. Cited by {cited_by_count} works.]"
85
+
86
+ # Extract authors (limit to 5)
87
+ authors = self._extract_authors(work.get("authorships", []))
88
+
89
+ # Extract concepts (top 5 by score)
90
+ concepts = self._extract_concepts(work.get("concepts", []))
91
+
92
+ # Open access info
93
+ oa_info = work.get("open_access", {})
94
+ is_oa = oa_info.get("is_oa", False)
95
+
96
+ # Get PDF URL (prefer best_oa_location)
97
+ best_oa = work.get("best_oa_location", {})
98
+ pdf_url = best_oa.get("pdf_url") if best_oa else None
99
+
100
+ # Build URL
101
+ if doi:
102
+ url = doi if doi.startswith("http") else f"https://doi.org/{doi}"
103
+ else:
104
+ openalex_id = work.get("id", "")
105
+ url = openalex_id if openalex_id else "https://openalex.org"
106
+
107
+ # Prepend citation badge to content
108
+ citation_badge = f"[Cited by {cited_by_count}] " if cited_by_count > 0 else ""
109
+ content = f"{citation_badge}{abstract[:1900]}"
110
+
111
+ # Calculate relevance: normalized citation count (capped at 1.0 for 100 citations)
112
+ # 100 citations is a very strong signal in most fields.
113
+ relevance = min(1.0, cited_by_count / 100.0)
114
+
115
+ return Evidence(
116
+ content=content[:2000],
117
+ citation=Citation(
118
+ source="openalex",
119
+ title=title[:500],
120
+ url=url,
121
+ date=str(year),
122
+ authors=authors,
123
+ ),
124
+ relevance=relevance,
125
+ metadata={
126
+ "cited_by_count": cited_by_count,
127
+ "concepts": concepts,
128
+ "is_open_access": is_oa,
129
+ "pdf_url": pdf_url,
130
+ },
131
+ )
132
+
133
+ def _reconstruct_abstract(self, inverted_index: dict[str, list[int]] | None) -> str:
134
+ """Rebuild abstract from {"word": [positions]} format."""
135
+ if not inverted_index:
136
+ return ""
137
+
138
+ position_word: dict[int, str] = {}
139
+ for word, positions in inverted_index.items():
140
+ for pos in positions:
141
+ position_word[pos] = word
142
+
143
+ if not position_word:
144
+ return ""
145
+
146
+ max_pos = max(position_word.keys())
147
+ return " ".join(position_word.get(i, "") for i in range(max_pos + 1))
148
+
149
+ def _extract_authors(self, authorships: list[dict[str, Any]]) -> list[str]:
150
+ """Extract author names from authorships array."""
151
+ authors = []
152
+ for authorship in authorships[:5]:
153
+ author = authorship.get("author", {})
154
+ name = author.get("display_name")
155
+ if name:
156
+ authors.append(name)
157
+ return authors
158
+
159
+ def _extract_concepts(self, concepts: list[dict[str, Any]]) -> list[str]:
160
+ """Extract concept names, sorted by score."""
161
+ sorted_concepts = sorted(concepts, key=lambda c: c.get("score", 0), reverse=True)
162
+ return [c.get("display_name", "") for c in sorted_concepts[:5] if c.get("display_name")]
tests/unit/tools/test_openalex.py ADDED
@@ -0,0 +1,165 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Unit tests for OpenAlex tool."""
2
+
3
+ from unittest.mock import AsyncMock, MagicMock
4
+
5
+ import pytest
6
+
7
+ from src.tools.openalex import OpenAlexTool
8
+ from src.utils.models import Evidence
9
+
10
+ # Sample OpenAlex response
11
+ SAMPLE_OPENALEX_RESPONSE = {
12
+ "results": [
13
+ {
14
+ "id": "https://openalex.org/W12345",
15
+ "doi": "https://doi.org/10.1234/test",
16
+ "display_name": "Metformin in Cancer Treatment",
17
+ "publication_year": 2024,
18
+ "cited_by_count": 150,
19
+ "abstract_inverted_index": {
20
+ "Metformin": [0],
21
+ "shows": [1],
22
+ "promise": [2],
23
+ "in": [3],
24
+ "cancer": [4],
25
+ "treatment": [5],
26
+ },
27
+ "concepts": [
28
+ {"display_name": "Metformin", "score": 0.95, "level": 2},
29
+ {"display_name": "Cancer", "score": 0.88, "level": 1},
30
+ ],
31
+ "authorships": [
32
+ {"author": {"display_name": "John Smith"}},
33
+ {"author": {"display_name": "Jane Doe"}},
34
+ ],
35
+ "open_access": {"is_oa": True, "oa_url": "https://example.com/oa"},
36
+ "best_oa_location": {"pdf_url": "https://example.com/paper.pdf"},
37
+ }
38
+ ]
39
+ }
40
+
41
+
42
+ @pytest.mark.unit
43
+ class TestOpenAlexTool:
44
+ """Tests for OpenAlexTool."""
45
+
46
+ @pytest.fixture
47
+ def tool(self) -> OpenAlexTool:
48
+ return OpenAlexTool()
49
+
50
+ @pytest.fixture
51
+ def mock_client(self, mocker):
52
+ """Create a standardized mock client with context manager support."""
53
+ client = AsyncMock()
54
+ client.__aenter__.return_value = client
55
+ client.__aexit__.return_value = None
56
+
57
+ # Standard response mock
58
+ resp = MagicMock()
59
+ resp.json.return_value = SAMPLE_OPENALEX_RESPONSE
60
+ resp.raise_for_status.return_value = None
61
+ client.get.return_value = resp
62
+
63
+ mocker.patch("httpx.AsyncClient", return_value=client)
64
+ return client
65
+
66
+ def test_tool_name(self, tool: OpenAlexTool) -> None:
67
+ """Tool name should be 'openalex'."""
68
+ assert tool.name == "openalex"
69
+
70
+ @pytest.mark.asyncio
71
+ async def test_search_returns_evidence(self, tool: OpenAlexTool, mock_client) -> None:
72
+ """Search should return Evidence objects."""
73
+ results = await tool.search("metformin cancer", max_results=5)
74
+
75
+ assert len(results) == 1
76
+ assert isinstance(results[0], Evidence)
77
+ assert results[0].citation.source == "openalex"
78
+
79
+ @pytest.mark.asyncio
80
+ async def test_search_includes_citation_count(self, tool: OpenAlexTool, mock_client) -> None:
81
+ """Evidence metadata should include cited_by_count."""
82
+ results = await tool.search("metformin cancer", max_results=5)
83
+ assert results[0].metadata["cited_by_count"] == 150
84
+
85
+ @pytest.mark.asyncio
86
+ async def test_search_calculates_relevance(self, tool: OpenAlexTool, mock_client) -> None:
87
+ """Evidence relevance should be based on citations (capped at 1.0)."""
88
+ results = await tool.search("metformin cancer", max_results=5)
89
+ # 150 citations / 100 = 1.5 -> capped at 1.0
90
+ assert results[0].relevance == 1.0
91
+
92
+ @pytest.mark.asyncio
93
+ async def test_search_includes_concepts(self, tool: OpenAlexTool, mock_client) -> None:
94
+ """Evidence metadata should include concepts."""
95
+ results = await tool.search("metformin cancer", max_results=5)
96
+ assert "Metformin" in results[0].metadata["concepts"]
97
+ assert "Cancer" in results[0].metadata["concepts"]
98
+
99
+ @pytest.mark.asyncio
100
+ async def test_search_includes_open_access_info(self, tool: OpenAlexTool, mock_client) -> None:
101
+ """Evidence metadata should include open access info."""
102
+ results = await tool.search("metformin cancer", max_results=5)
103
+ assert results[0].metadata["is_open_access"] is True
104
+ assert results[0].metadata["pdf_url"] == "https://example.com/paper.pdf"
105
+
106
+ def test_reconstruct_abstract(self, tool: OpenAlexTool) -> None:
107
+ """Abstract reconstruction from inverted index."""
108
+ inverted_index = {
109
+ "Hello": [0],
110
+ "world": [1],
111
+ "this": [2],
112
+ "is": [3],
113
+ "a": [4],
114
+ "test": [5],
115
+ }
116
+ result = tool._reconstruct_abstract(inverted_index)
117
+ assert result == "Hello world this is a test"
118
+
119
+ def test_reconstruct_abstract_empty(self, tool: OpenAlexTool) -> None:
120
+ """Handle None or empty inverted index."""
121
+ assert tool._reconstruct_abstract(None) == ""
122
+ assert tool._reconstruct_abstract({}) == ""
123
+
124
+ @pytest.mark.asyncio
125
+ async def test_search_empty_results(self, tool: OpenAlexTool, mock_client) -> None:
126
+ """Handle empty results gracefully."""
127
+ mock_client.get.return_value.json.return_value = {"results": []}
128
+
129
+ results = await tool.search("xyznonexistent123", max_results=5)
130
+
131
+ assert results == []
132
+
133
+ @pytest.mark.asyncio
134
+ async def test_search_params(self, tool: OpenAlexTool, mock_client) -> None:
135
+ """Verify API call requests citation-sorted results and uses polite pool."""
136
+ mock_client.get.return_value.json.return_value = {"results": []}
137
+
138
+ await tool.search("test query", max_results=5)
139
+
140
+ # Verify call params
141
+ call_args = mock_client.get.call_args
142
+ params = call_args[1]["params"]
143
+ assert params["sort"] == "cited_by_count:desc"
144
+ assert params["mailto"] == tool.POLITE_EMAIL
145
+ assert "type:article" in params["filter"]
146
+ assert "has_abstract:true" in params["filter"]
147
+
148
+
149
+ @pytest.mark.integration
150
+ class TestOpenAlexIntegration:
151
+ """Integration tests with real OpenAlex API."""
152
+
153
+ @pytest.mark.asyncio
154
+ async def test_real_api_returns_results(self) -> None:
155
+ """Test actual API returns relevant results."""
156
+ tool = OpenAlexTool()
157
+ results = await tool.search("metformin cancer treatment", max_results=3)
158
+
159
+ assert len(results) > 0
160
+ # Should have citation counts
161
+ assert results[0].metadata["cited_by_count"] >= 0
162
+ # Should have abstract text
163
+ assert len(results[0].content) > 50
164
+ # Should have concepts
165
+ assert len(results[0].metadata["concepts"]) > 0