Update search_pipeline.py

search_pipeline.py

@@ -168,11 +168,7 @@ async def scrape_target_url(client, result: dict):
     url = result["href"]
 
     try:
-        response = await client.get(
-            url,
-            timeout=12.0,
-            follow_redirects=True,
-        )
+       response = await client.get(url, timeout=12.0)
 
         if response.status_code != 200:
             logger.warning(
@@ -276,7 +272,7 @@ async def build_context_pipeline(
 
             scraped = await asyncio.gather(
                 *tasks,
-                return_exceptions=False,
+                return_exceptions=True,
             )
     else:
         async with httpx.AsyncClient(
@@ -290,7 +286,7 @@ async def build_context_pipeline(
 
             scraped = await asyncio.gather(
                 *tasks,
-                return_exceptions=False,
+                return_exceptions=True,
             )
 
     blocks = []