Spaces:
Sleeping
Sleeping
File size: 3,185 Bytes
e820e26 80ea88d e820e26 80ea88d e820e26 80ea88d e820e26 80ea88d e820e26 80ea88d e820e26 80ea88d e820e26 80ea88d e820e26 80ea88d e820e26 80ea88d e820e26 80ea88d e820e26 80ea88d e820e26 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 |
# import time
# from typing import Optional
# import requests
# from bs4 import BeautifulSoup
# from langchain.tools import tool
# class WebSearchTool:
# def __init__(self):
# self.last_request_time = 0
# self.min_request_interval = 2.0 # Minimum time between requests in seconds
# self.max_retries = 10
# def search(self, query: str, domain: Optional[str] = None) -> str:
# """Perform web search with rate limiting and retries."""
# for attempt in range(self.max_retries):
# # Implement rate limiting
# current_time = time.time()
# time_since_last = current_time - self.last_request_time
# if time_since_last < self.min_request_interval:
# time.sleep(self.min_request_interval - time_since_last)
# try:
# # Make the search request
# results = self._do_search(query, domain)
# self.last_request_time = time.time()
# return results
# except Exception as e:
# if "202 Ratelimit" in str(e):
# if attempt < self.max_retries - 1:
# # Exponential backoff
# wait_time = (2 ** attempt) * self.min_request_interval
# time.sleep(wait_time)
# continue
# return f"Search failed after {self.max_retries} attempts: {str(e)}"
# return "Search failed due to rate limiting"
# def _do_search(self, query: str, domain: Optional[str] = None) -> str:
# """Perform the actual search request."""
# try:
# # Construct search URL
# base_url = "https://html.duckduckgo.com/html"
# params = {"q": query}
# if domain:
# params["q"] += f" site:{domain}"
# # Make request with increased timeout
# response = requests.get(base_url, params=params, timeout=10)
# response.raise_for_status()
# if response.status_code == 202:
# raise Exception("202 Ratelimit")
# # Extract search results
# results = []
# soup = BeautifulSoup(response.text, 'html.parser')
# for result in soup.find_all('div', {'class': 'result'}):
# title = result.find('a', {'class': 'result__a'})
# snippet = result.find('a', {'class': 'result__snippet'})
# if title and snippet:
# results.append({
# 'title': title.get_text(),
# 'snippet': snippet.get_text(),
# 'url': title.get('href')
# })
# # Format results
# formatted_results = []
# for r in results[:10]: # Limit to top 5 results
# formatted_results.append(f"[{r['title']}]({r['url']})\n{r['snippet']}\n")
# return "## Search Results\n\n" + "\n".join(formatted_results)
# except requests.RequestException as e:
# raise Exception(f"Search request failed: {str(e)}") |