Update main.py

main.py

@@ -10,11 +10,12 @@ from concurrent.futures import ThreadPoolExecutor
 from webscout.litagent import LitAgent
 from bs4 import BeautifulSoup
 import json
+import xml.etree.ElementTree as ET # Added for Google Suggestions XML parsing
 
 app = FastAPI(
-    title="Bing Search API",
-    description="A FastAPI wrapper for the BingSearch library with advanced features.",
-    version="1.0.0",
+    title="Web Search API",
+    description="A FastAPI wrapper for Bing and Google search services.",
+    version="1.1.0",
 )
 
 # --- BingSearch Library Code ---
@@ -64,16 +65,14 @@ class BingSearch:
             timeout=self.timeout,
             impersonate=impersonate
         )
-        # It's good practice to set a realistic User-Agent
         self.session.headers.update({
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36"
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/536.36"
         })
 
-    # FIX: Updated selectors to be more robust against Bing UI changes.
     def _selectors(self, element):
         selectors = {
-            'links': 'ol#b_results > li',  # More generic selector for any list item in results
-            'next': 'a.sb_pagN' # Selector for the "Next" page button
+            'links': 'ol#b_results > li',
+            'next': 'a.sb_pagN'
         }
         return selectors[element]
 
@@ -91,6 +90,10 @@ class BingSearch:
 
     def _get_url(self, tag):
         url = tag.get('href', '')
+        # FIX: Ensure relative URLs from Bing (like /news/search) become absolute
+        if url.startswith('/'):
+            return self._base_url + url
+        
         resp = url
         try:
             parsed_url = urlparse(url)
@@ -107,7 +110,6 @@ class BingSearch:
             print(f"Error decoding Base64 string: {e}")
         return resp
 
-    # FIX: The entire text parsing logic is updated to handle modern Bing HTML structure.
     def text(
         self,
         keywords: str,
@@ -136,11 +138,9 @@ class BingSearch:
             html = fetch_page(current_url)
             soup = BeautifulSoup(html, "html.parser")
             
-            # Use the more generic selector for result blocks
             result_blocks = soup.select(self._selectors('links'))
             
             for result in result_blocks:
-                # Find the title and link, which are usually in an <h2> tag
                 title_tag = result.find('h2')
                 if not title_tag:
                     continue
@@ -150,24 +150,19 @@ class BingSearch:
                     continue
                 
                 url_val = self._get_url(link_tag)
-                title = title_tag.get_text(strip=True)
+                # FIX: Use separator=' ' to prevent text from being squashed together.
+                title = title_tag.get_text(strip=True, separator=' ')
 
-                # Find the description, often in a div with class 'b_caption'
                 desc_container = result.find('div', class_='b_caption')
                 description = ''
                 if desc_container:
-                    # Find the paragraph within the caption, or use the whole caption text
-                    desc_p = desc_container.find('p')
-                    if desc_p:
-                        description = desc_p.get_text(strip=True)
-                    else:
-                        description = desc_container.get_text(strip=True)
+                    # FIX: Use separator=' ' for descriptions as well.
+                    description = desc_container.get_text(strip=True, separator=' ')
                 
-                # Fallback if no 'b_caption' is found
                 if not description:
                     p_tag = result.find('p')
                     if p_tag:
-                        description = p_tag.get_text(strip=True)
+                        description = p_tag.get_text(strip=True, separator=' ')
 
                 if url_val and title:
                     if unique and url_val in fetched_links:
@@ -182,7 +177,6 @@ class BingSearch:
             if len(fetched_results) >= max_results:
                 break
 
-            # Find the next page URL
             next_page_info = self._next_page(soup)
             current_url = next_page_info['url']
             if current_url:
@@ -190,7 +184,6 @@ class BingSearch:
 
         return fetched_results[:max_results]
 
-
     def suggestions(self, query: str, region: str = None) -> List[str]:
         if not query:
             raise ValueError("Search query cannot be empty")
@@ -207,10 +200,7 @@ class BingSearch:
                 return data[1]
             return []
         except Exception as e:
-            if hasattr(e, 'response') and e.response is not None:
-                raise Exception(f"Bing suggestions failed with status {e.response.status_code}: {str(e)}")
-            else:
-                raise Exception(f"Bing suggestions failed: {str(e)}")
+            raise Exception(f"Bing suggestions failed: {str(e)}")
 
     def images(
         self,
@@ -221,18 +211,9 @@ class BingSearch:
     ) -> List[BingImageResult]:
         if not keywords:
             raise ValueError("Search keywords cannot be empty")
-        safe_map = {
-            "on": "Strict",
-            "moderate": "Moderate",
-            "off": "Off"
-        }
+        safe_map = {"on": "Strict", "moderate": "Moderate", "off": "Off"}
         safe = safe_map.get(safesearch.lower(), "Moderate")
-        params = {
-            "q": keywords,
-            "count": max_results,
-            "setlang": self.lang,
-            "safeSearch": safe,
-        }
+        params = {"q": keywords, "count": max_results, "setlang": self.lang, "safeSearch": safe}
         if region:
             params["mkt"] = region
         url = f"{self._base_url}/images/search?{urlencode(params)}"
@@ -241,10 +222,7 @@ class BingSearch:
             resp.raise_for_status()
             html = resp.text
         except Exception as e:
-            if hasattr(e, 'response') and e.response is not None:
-                raise Exception(f"Bing image search failed with status {e.response.status_code}: {str(e)}")
-            else:
-                raise Exception(f"Bing image search failed: {str(e)}")
+            raise Exception(f"Bing image search failed: {str(e)}")
         soup = BeautifulSoup(html, "html.parser")
         results = []
         for item in soup.select("a.iusc"):
@@ -273,17 +251,9 @@ class BingSearch:
     ) -> List['BingNewsResult']:
         if not keywords:
             raise ValueError("Search keywords cannot be empty")
-        safe_map = {
-            "on": "Strict",
-            "moderate": "Moderate",
-            "off": "Off"
-        }
+        safe_map = {"on": "Strict", "moderate": "Moderate", "off": "Off"}
         safe = safe_map.get(safesearch.lower(), "Moderate")
-        params = {
-            "q": keywords,
-            "form": "QBNH",
-            "safeSearch": safe,
-        }
+        params = {"q": keywords, "form": "QBNH", "safeSearch": safe}
         if region:
             params["mkt"] = region
         url = f"{self._base_url}/news/search?{urlencode(params)}"
@@ -291,27 +261,24 @@ class BingSearch:
             resp = self.session.get(url)
             resp.raise_for_status()
         except Exception as e:
-            if hasattr(e, 'response') and e.response is not None:
-                raise Exception(f"Bing news search failed with status {e.response.status_code}: {str(e)}")
-            else:
-                raise Exception(f"Bing news search failed: {str(e)}")
+            raise Exception(f"Bing news search failed: {str(e)}")
         soup = BeautifulSoup(resp.text, "html.parser")
         results = []
         for item in soup.select("div.news-card, div.card, div.newsitem, div.card-content, div.t_s_main"):
             a_tag = item.find("a")
-            title = a_tag.get_text(strip=True) if a_tag else ''
+            title = a_tag.get_text(strip=True, separator=' ') if a_tag else ''
             url_val = a_tag['href'] if a_tag and a_tag.has_attr('href') else ''
             desc_tag = item.find("div", class_="snippet") or item.find("div", class_="news-card-snippet") or item.find("div", class_="snippetText")
-            description = desc_tag.get_text(strip=True) if desc_tag else ''
+            description = desc_tag.get_text(strip=True, separator=' ') if desc_tag else ''
             source_tag = item.find("div", class_="source")
-            source = source_tag.get_text(strip=True) if source_tag else ''
+            source = source_tag.get_text(strip=True, separator=' ') if source_tag else ''
             if url_val and title:
                 results.append(BingNewsResult(title=title, url=url_val, description=description, source=source))
                 if len(results) >= max_results:
                     break
         if not results:
             for item in soup.select("a.title"):
-                title = item.get_text(strip=True)
+                title = item.get_text(strip=True, separator=' ')
                 url_val = item['href'] if item.has_attr('href') else ''
                 description = ''
                 source = ''
@@ -321,83 +288,107 @@ class BingSearch:
                         break
         return results[:max_results]
 
+# --- NEW: Google Suggestions Class ---
+class GoogleSuggestions:
+    """Fetches search suggestions from Google's unofficial suggestions API."""
+    def __init__(self, timeout: int = 10, proxies: Optional[Dict[str, str]] = None):
+        self.session = Session(
+            proxies=proxies if proxies else {},
+            timeout=timeout,
+            impersonate="chrome110"
+        )
+        self.session.headers.update({
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36"
+        })
+
+    def get(self, query: str, lang: str = "en") -> List[str]:
+        if not query:
+            raise ValueError("Search query cannot be empty")
+            
+        params = {"q": query, "hl": lang, "output": "toolbar"}
+        url = f"https://clients1.google.com/complete/search?{urlencode(params)}"
+        
+        try:
+            response = self.session.get(url)
+            response.raise_for_status()
+            
+            suggestions = []
+            root = ET.fromstring(response.content)
+            for suggestion_node in root.findall('CompleteSuggestion/suggestion'):
+                data = suggestion_node.get('data')
+                if data:
+                    suggestions.append(data)
+            return suggestions
+        except Exception as e:
+            raise Exception(f"Google suggestions failed: {str(e)}")
+
+# --- FastAPI Endpoints ---
 
 bing = BingSearch()
+google = GoogleSuggestions()
 
-@app.get("/search", response_model=List[BingSearchResult])
-async def text_search(
+@app.get("/bing/search", response_model=List[BingSearchResult], tags=["Bing"])
+async def bing_text_search(
     query: str = Query(..., description="The search keywords."),
     region: Optional[str] = Query(None, description="The region for the search (e.g., 'us-US')."),
     safesearch: str = Query("moderate", description="Safe search level ('on', 'moderate', 'off')."),
     max_results: int = Query(10, description="Maximum number of results to return."),
 ):
-    """
-    Perform a text search on Bing.
-    """
+    """Perform a text search on Bing."""
     try:
-        results = bing.text(
-            keywords=query,
-            region=region,
-            safesearch=safesearch,
-            max_results=max_results,
-        )
-        return results
+        return bing.text(keywords=query, region=region, safesearch=safesearch, max_results=max_results)
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
 
-@app.get("/suggestions", response_model=List[str])
-async def get_suggestions(
+@app.get("/bing/suggestions", response_model=List[str], tags=["Bing"])
+async def bing_get_suggestions(
     query: str = Query(..., description="The search query for which to fetch suggestions."),
     region: Optional[str] = Query(None, description="The region for the suggestions (e.g., 'en-US')."),
 ):
-    """
-    Fetches search suggestions for a given query.
-    """
+    """Fetches search suggestions for a given query from Bing."""
     try:
-        suggestions = bing.suggestions(query=query, region=region)
-        return suggestions
+        return bing.suggestions(query=query, region=region)
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
 
-@app.get("/images", response_model=List[BingImageResult])
-async def image_search(
+@app.get("/bing/images", response_model=List[BingImageResult], tags=["Bing"])
+async def bing_image_search(
     query: str = Query(..., description="The search keywords for images."),
     region: Optional[str] = Query(None, description="The region for the image search (e.g., 'us-US')."),
     safesearch: str = Query("moderate", description="Safe search level ('on', 'moderate', 'off')."),
     max_results: int = Query(10, description="Maximum number of image results to return."),
 ):
-    """
-    Perform an image search on Bing.
-    """
+    """Perform an image search on Bing."""
     try:
-        results = bing.images(
-            keywords=query,
-            region=region,
-            safesearch=safesearch,
-            max_results=max_results,
-        )
-        return results
+        return bing.images(keywords=query, region=region, safesearch=safesearch, max_results=max_results)
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
 
-@app.get("/news", response_model=List[BingNewsResult])
-async def news_search(
+@app.get("/bing/news", response_model=List[BingNewsResult], tags=["Bing"])
+async def bing_news_search(
     query: str = Query(..., description="The search keywords for news."),
     region: Optional[str] = Query(None, description="The region for the news search (e.g., 'us-US')."),
     safesearch: str = Query("moderate", description="Safe search level ('on', 'moderate', 'off')."),
     max_results: int = Query(10, description="Maximum number of news results to return."),
+):
+    """Perform a news search on Bing."""
+    try:
+        return bing.news(keywords=query, region=region, safesearch=safesearch, max_results=max_results)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+# --- NEW: Google Suggestions Endpoint ---
+@app.get("/google/suggestions", response_model=List[str], tags=["Google"])
+async def google_get_suggestions(
+    query: str = Query(..., description="The search query for which to fetch suggestions."),
+    lang: str = Query("en", description="The language for the suggestions (e.g., 'en', 'es')."),
 ):
     """
-    Perform a news search on Bing.
+    Fetches search suggestions for a given query from Google.
+    This replicates the functionality of the provided C++ code.
     """
     try:
-        results = bing.news(
-            keywords=query,
-            region=region,
-            safesearch=safesearch,
-            max_results=max_results,
-        )
-        return results
+        return google.get(query=query, lang=lang)
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))