Update search_pipeline.py

search_pipeline.py

@@ -163,15 +163,20 @@ async def search_web_async(query: str) -> list:
 # ---------------------------------------------------------------------
 # WEB SCRAPING
 # ---------------------------------------------------------------------
-
 async def scrape_target_url(client, result: dict):
     url = result["href"]
 
     try:
+        # Check if using curl_cffi or standard httpx
+        if "curl_cffi" in str(type(client)):
+            kwargs = {"allow_redirects": True}
+        else:
+            kwargs = {"follow_redirects": True}
+
         response = await client.get(
             url,
             timeout=12.0,
-            follow_redirects=True,
+            **kwargs,
         )
 
         if response.status_code != 200: