feat: start writing tools

src/deepengineer/webcrawler/async_search.py

@@ -93,7 +93,7 @@ def get_linkup_balance():
     return balance
 
 
-async def async_linkup_search(
+async def linkup_search_async(
     search_query: str,
     depth: Literal["standard", "deep"] = "standard",
     output_type: Literal['searchResults', 'sourcedAnswer', 'structured'] = "sourcedAnswer",
@@ -138,24 +138,24 @@ async def async_linkup_search(
 async def arxiv_search_async(
     search_query: str,
 ) -> SearchResponse:
-    response = await async_linkup_search(search_query, include_domains=[ScientificDomains.arxiv])
+    response = await linkup_search_async(search_query, include_domains=[ScientificDomains.arxiv])
     return response
 
 
 async def pubmed_search_async(
     search_query: str,
 ) -> SearchResponse:
-    response = await async_linkup_search(search_query, include_domains=[ScientificDomains.pubmed])
+    response = await linkup_search_async(search_query, include_domains=[ScientificDomains.pubmed])
     return response
 
 async def sciencedirect_search_async(
     search_query: str,
 ) -> SearchResponse:
-    response = await async_linkup_search(search_query, include_domains=[ScientificDomains.sciencedirect])
+    response = await linkup_search_async(search_query, include_domains=[ScientificDomains.sciencedirect])
     return response
 
 async def scientific_search_async(
     search_query: str,
 ) -> SearchResponse:
-    response = await async_linkup_search(search_query, include_domains=[ScientificDomains.wikipedia, ScientificDomains.arxiv, ScientificDomains.pubmed, ScientificDomains.sciencedirect])
+    response = await linkup_search_async(search_query, include_domains=[ScientificDomains.wikipedia, ScientificDomains.arxiv, ScientificDomains.pubmed, ScientificDomains.sciencedirect])
     return response

src/deepengineer/webcrawler/tools.py

@@ -0,0 +1,14 @@
+from deepengineer.webcrawler.async_search import linkup_search_async, tavily_search_async, arxiv_search_async, pubmed_search_async, sciencedirect_search_async, scientific_search_async
+from deepengineer.webcrawler.async_crawl import crawl4ai_extract_markdown_of_url_async, arxiv_download_pdf_async, download_pdf_async
+
+
+
+
+
+
+
+
+
+
+
+

tests/webcrawler/test_async_crawl.py

@@ -5,7 +5,6 @@ from deepengineer.webcrawler.async_crawl import (
     arxiv_download_pdf_async,
 )
 from mistralai import OCRResponse
-from deepengineer.webcrawler.pdf_tools import convert_pdf_to_markdown_async
 from deepengineer.webcrawler.testing import URL_WIKIPEDIA, URL_PDF, ARXIV_URL
 from deepengineer.common_path import DATA_DIR
 
@@ -31,14 +30,3 @@ async def test_arxiv_download_pdf_async():
     pdf_path = await arxiv_download_pdf_async(ARXIV_URL, output_path=output_path)
     assert pdf_path == output_path
     assert output_path.exists()
-
-@pytest.mark.expensive
-@pytest.mark.asyncio
-async def test_convert_pdf_to_markdown_async():
-    pdf_path = DATA_DIR / "report_thermal_neutron.pdf"
-    assert pdf_path.exists()
-    markdown, ocr_response = await convert_pdf_to_markdown_async(pdf_path)
-    assert isinstance(ocr_response, OCRResponse)
-    assert len(ocr_response.pages) == 16
-    assert isinstance(markdown, str)
-    assert "where each cylinder represent" in markdown

tests/webcrawler/test_async_search.py

@@ -4,7 +4,7 @@ from deepengineer.webcrawler.async_search import (
     tavily_search_async,
     SearchResponse,
     get_tavily_usage,
-    async_linkup_search,
+    linkup_search_async,
     get_linkup_balance
 )
 
@@ -47,7 +47,7 @@ async def test_linkup_search_async():
     balance_before = get_linkup_balance()
     print(balance_before)
 
-    response = await async_linkup_search(
+    response = await linkup_search_async(
         search_query="Would it be possible to make a thermal reactor with graphite and lead?",
     )
     print(response.answer)

tests/webcrawler/test_pdfs_tools.py

@@ -0,0 +1,16 @@
+from deepengineer.webcrawler.pdf_tools import convert_pdf_to_markdown_async
+from mistralai import OCRResponse
+from deepengineer.common_path import DATA_DIR
+import pytest
+
+
+@pytest.mark.expensive
+@pytest.mark.asyncio
+async def test_convert_pdf_to_markdown_async():
+    pdf_path = DATA_DIR / "report_thermal_neutron.pdf"
+    assert pdf_path.exists()
+    markdown, ocr_response = await convert_pdf_to_markdown_async(pdf_path)
+    assert isinstance(ocr_response, OCRResponse)
+    assert len(ocr_response.pages) == 16
+    assert isinstance(markdown, str)
+    assert "where each cylinder represent" in markdown