update

app/services/tools.py

@@ -3,16 +3,10 @@ import logging
 import os
 from pathlib import Path
 from typing import Any, Dict, List, Optional
-
 from PIL import Image
-
 from app.services.vector_database_search import VectorDatabaseSearch
 from app.services.websearch import WebSearch
 from MagicConvert import MagicConvert
-from app.services.prompts import (
-    ADVICE_REPORT_SUGGESTION,
-    URDU_ADVICE_REPORT_SUGGESTION,
-)
 from app.services.image_classification_vit import SkinDiseaseClassifier
 
 try:

app/services/websearch.py

@@ -5,7 +5,8 @@ from bs4 import BeautifulSoup
 import urllib.parse
 import time
 import random
-from urllib.parse import urlparse, parse_qs
+from urllib.parse import urlparse, parse_qs, unquote
+from concurrent.futures import ThreadPoolExecutor, as_completed
 
 warnings.simplefilter('ignore', requests.packages.urllib3.exceptions.InsecureRequestWarning)
 
@@ -113,35 +114,33 @@ class WebSearch:
             # If any error occurs, return the original URL
             return url
 
-    def extract_real_url_from_redirect(self, url):
-        """Extract the actual URL from a redirect URL"""
+    def extract_url_from_duckduckgo(self, ddg_link):
+        """Simplified extraction of actual URL from DuckDuckGo redirect"""
         try:
-            parsed = urlparse(url)
+            # Parse the DuckDuckGo URL
+            parsed = urlparse(ddg_link)
             
-            # Handle DuckDuckGo redirects
-            if "duckduckgo.com" in parsed.netloc and "u3=" in parsed.query:
-                params = parse_qs(parsed.query)
-                if "u3" in params and params["u3"]:
-                    redirect_url = params["u3"][0]
-                    # Handle nested redirects (like Bing redirects inside DuckDuckGo)
-                    if "bing.com/aclick" in redirect_url:
-                        bing_parsed = urlparse(redirect_url)
-                        bing_params = parse_qs(bing_parsed.query)
-                        if "u" in bing_params and bing_params["u"]:
-                            decoded_url = urllib.parse.unquote(bing_params["u"][0])
-                            return self.clean_url(decoded_url)
-                    return self.clean_url(redirect_url)
+            # Get the uddg parameter which contains the actual URL
+            params = parse_qs(parsed.query)
+            uddg = params.get('uddg', [''])[0]
             
-            # Handle Bing redirects
-            if "bing.com/aclick" in url:
-                params = parse_qs(parsed.query)
-                if "u" in params and params["u"]:
-                    return self.clean_url(urllib.parse.unquote(params["u"][0]))
-                    
-            return url
+            # Decode the URL
+            if uddg:
+                actual_url = unquote(uddg)
+            else:
+                # Fallback to original if no uddg parameter
+                actual_url = ddg_link
             
-        except Exception:
-            return url
+            # Ensure proper URL format
+            if actual_url and not actual_url.startswith(("http://", "https://")):
+                actual_url = "https://" + actual_url
+            
+            # Clean the URL from tracking parameters
+            return self.clean_url(actual_url)
+            
+        except Exception as e:
+            print(f"Error extracting URL: {e}")
+            return ddg_link
 
     def extract_text_from_webpage(self, html_content):
         soup = BeautifulSoup(html_content, "html.parser")
@@ -159,34 +158,42 @@ class WebSearch:
 
     def search(self, query):
         results = []
-        encoded_query = urllib.parse.quote(query)
-        url = f'https://html.duckduckgo.com/html/?q={encoded_query}'
+        
+        # Sanitize query
+        query = re.sub(r'[<>"\']', "", query.strip())
+        if not query:
+            print("Empty search query")
+            return []
+        
+        # Setup request parameters
+        params = {"q": query, "kl": "us-en"}
+        url = "https://html.duckduckgo.com/html/"
 
         try:
             with requests.Session() as session:
                 session.headers.update(self.headers)
 
-                response = session.get(url, timeout=10)
+                # Make search request
+                response = session.get(url, params=params, timeout=10)
+                response.raise_for_status()
+                
+                # Parse results
                 soup = BeautifulSoup(response.text, 'html.parser')
                 
-                # Getting more results than needed to account for filtering
-                search_results = soup.find_all('div', class_='result')[:self.num_results * 2]
+                # Extract links using simplified approach
                 links = []
-                
-                # Extract and process links
-                for result in search_results:
-                    link_tag = result.find('a', class_='result__a')
-                    if not link_tag or not link_tag.get('href'):
-                        continue
-                        
-                    original_link = link_tag['href']
-                    
-                    # Process link to get the actual URL
-                    clean_link = self.extract_real_url_from_redirect(original_link)
-                    
-                    # Validate the URL
-                    if self.is_valid_url(clean_link):
-                        links.append(clean_link)
+                for result in soup.select("div.result"):
+                    title_tag = result.select_one("a.result__a")
+                    if title_tag:
+                        # Get the href attribute
+                        raw_link = title_tag.get("href", "")
+                        if raw_link:
+                            # Extract actual URL from DuckDuckGo redirect
+                            actual_url = self.extract_url_from_duckduckgo(raw_link)
+                            
+                            # Validate the URL
+                            if self.is_valid_url(actual_url):
+                                links.append(actual_url)
                 
                 # Prioritize content domains
                 prioritized_links = []
@@ -211,8 +218,7 @@ class WebSearch:
                         unique_links.append(link)
                         seen_domains.add(domain)
 
-                from concurrent.futures import ThreadPoolExecutor, as_completed
-
+                # Fetch page content
                 def fetch_page(link):
                     try:
                         # Random delay to avoid being blocked

ddg_response.html

@@ -0,0 +1,176 @@
+<!-- <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd"> -->
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <link rel="canonical" href="https://duckduckgo.com/">
+    <meta http-equiv="content-type" content="text/html; charset=UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=3.0, user-scalable=1">
+    <meta name="referrer" content="origin">
+    <title>
+        DuckDuckGo
+    </title>
+    <link rel="preload" href="/font/ProximaNova-Reg-webfont.woff2" as="font" type="font/woff2" crossorigin="anonymous">
+    <link rel="preload" href="/font/ProximaNova-Sbold-webfont.woff2" as="font" type="font/woff2" crossorigin="anonymous">
+    <link title="DuckDuckGo (Lite)" type="application/opensearchdescription+xml" rel="search" href="//duckduckgo.com/opensearch_lite_v2.xml">
+    <link rel="icon" href="//duckduckgo.com/favicon.ico" type="image/x-icon"/>
+    <link id="icon60" rel="apple-touch-icon" href="//duckduckgo.com/assets/icons/meta/DDG-iOS-icon_60x60.png?v=2"/>
+    <link id="icon76" rel="apple-touch-icon" sizes="76x76" href="//duckduckgo.com/assets/icons/meta/DDG-iOS-icon_76x76.png?v=2"/>
+    <link id="icon120" rel="apple-touch-icon" sizes="120x120" href="//duckduckgo.com/assets/icons/meta/DDG-iOS-icon_120x120.png?v=2"/>
+    <link id="icon152" rel="apple-touch-icon" sizes="152x152" href="//duckduckgo.com/assets/icons/meta/DDG-iOS-icon_152x152.png?v=2"/>
+    <link rel="image_src" href="//duckduckgo.com/assets/icons/meta/DDG-icon_256x256.png">
+    <link rel="stylesheet" media="handheld, all" href="//duckduckgo.com/dist/l.dedf9a21e97bacda4782.css" type="text/css"/>
+    <link rel="stylesheet" media="handheld, all" href="//duckduckgo.com/dist/lc.ddfa8c0ba43459a9159a.css" type="text/css"/>
+</head>
+
+<body>
+    <a name="top"></a>
+    <center id="lite_wrapper">
+        <br>
+        <a class="header-url" href="/html/">
+            <span class="header">DuckDuckGo</span>
+        </a>
+        <br><br>
+
+        
+        <iframe name="ifr" width="0" height="0" border="0" class="hidden"></iframe>
+        <form id="img-form" action="//duckduckgo.com/anomaly.js?sv=html&cc=botnet&ti=1758386895&gk=d4cd0dabcf4caa22ad92fab40844c786&p=d72c36ab4cb24f49969ecef4b2b88ad1-a3ec151498ae45aa845c5dda2f14a862-61b69bac6cc64d5886ca11ee9fc0eef2-d2cb2f084d1d45b6a98513a79b944bbc-5e6714da571b4c8895e1d949f47cedc2-23c1e3ae8ce8443183e624800acf2f47-ae7275bb00734944820af39a39d39107-d13828125211402eaef44405520c4080-aa87423535154d4f8f2a1cfff1de7a8b&q=acne&o=FYGE4bAiNVr8IcPNJsONTSwq441QyPzksuoqxWPyWuuL0OwnOoQUGNKgGm2X8RYI%0A&r=inc" target="ifr" method="POST"></form>
+        <form id="challenge-form" action="//duckduckgo.com/anomaly.js?sv=html&cc=botnet&st=1758386895&gk=d4cd0dabcf4caa22ad92fab40844c786&p=d72c36ab4cb24f49969ecef4b2b88ad1-a3ec151498ae45aa845c5dda2f14a862-61b69bac6cc64d5886ca11ee9fc0eef2-d2cb2f084d1d45b6a98513a79b944bbc-5e6714da571b4c8895e1d949f47cedc2-23c1e3ae8ce8443183e624800acf2f47-ae7275bb00734944820af39a39d39107-d13828125211402eaef44405520c4080-aa87423535154d4f8f2a1cfff1de7a8b&q=acne&o=FYGE4bAiNVr8IcPNJsONTSwq441QyPzksuoqxWPyWuuL0OwnOoQUGNKgGm2X8RYI%0A&r=inc" method="POST">
+            <div class="anomaly-modal__mask">
+                <div class="anomaly-modal__modal  is-ie" data-testid="anomaly-modal">
+                    <div class="anomaly-modal__title">Unfortunately, bots use DuckDuckGo too.</div>
+                    <div class="anomaly-modal__description">Please complete the following challenge to confirm this search was made by a human.</div>
+                    <div class="anomaly-modal__instructions">Select all squares containing a duck:</div>
+                    <div class="anomaly-modal__puzzle-margins">
+                        <div class="anomaly-modal__puzzle">
+                            
+                                <div class="anomaly-modal__box" data-index="0">
+                                    
+                                    
+                                        <label class="" for="image-check_d72c36ab4cb24f49969ecef4b2b88ad1" data-testid="anomaly-modal-tile-0">
+                                            <input type="checkbox" class="anomaly-modal__check" name="image-check_d72c36ab4cb24f49969ecef4b2b88ad1" id="image-check_d72c36ab4cb24f49969ecef4b2b88ad1">
+                                            <img class="anomaly-modal__image" alt=" " id="image-d72c36ab4cb24f49969ecef4b2b88ad1" src="../assets/anomaly/images/challenge/d72c36ab4cb24f49969ecef4b2b88ad1.jpg" data-id="d72c36ab4cb24f49969ecef4b2b88ad1.jpg" data-testid="anomaly-modal-image-0"></img>
+                                        </label>
+                                    
+                                </div>
+                            
+                                <div class="anomaly-modal__box" data-index="1">
+                                    
+                                    
+                                        <label class="" for="image-check_a3ec151498ae45aa845c5dda2f14a862" data-testid="anomaly-modal-tile-1">
+                                            <input type="checkbox" class="anomaly-modal__check" name="image-check_a3ec151498ae45aa845c5dda2f14a862" id="image-check_a3ec151498ae45aa845c5dda2f14a862">
+                                            <img class="anomaly-modal__image" alt=" " id="image-a3ec151498ae45aa845c5dda2f14a862" src="../assets/anomaly/images/challenge/a3ec151498ae45aa845c5dda2f14a862.jpg" data-id="a3ec151498ae45aa845c5dda2f14a862.jpg" data-testid="anomaly-modal-image-1"></img>
+                                        </label>
+                                    
+                                </div>
+                            
+                                <div class="anomaly-modal__box" data-index="2">
+                                    
+                                    
+                                        <label class="" for="image-check_61b69bac6cc64d5886ca11ee9fc0eef2" data-testid="anomaly-modal-tile-2">
+                                            <input type="checkbox" class="anomaly-modal__check" name="image-check_61b69bac6cc64d5886ca11ee9fc0eef2" id="image-check_61b69bac6cc64d5886ca11ee9fc0eef2">
+                                            <img class="anomaly-modal__image" alt=" " id="image-61b69bac6cc64d5886ca11ee9fc0eef2" src="../assets/anomaly/images/challenge/61b69bac6cc64d5886ca11ee9fc0eef2.jpg" data-id="61b69bac6cc64d5886ca11ee9fc0eef2.jpg" data-testid="anomaly-modal-image-2"></img>
+                                        </label>
+                                    
+                                </div>
+                            
+                                <div class="anomaly-modal__box" data-index="3">
+                                    
+                                    
+                                        <label class="" for="image-check_d2cb2f084d1d45b6a98513a79b944bbc" data-testid="anomaly-modal-tile-3">
+                                            <input type="checkbox" class="anomaly-modal__check" name="image-check_d2cb2f084d1d45b6a98513a79b944bbc" id="image-check_d2cb2f084d1d45b6a98513a79b944bbc">
+                                            <img class="anomaly-modal__image" alt=" " id="image-d2cb2f084d1d45b6a98513a79b944bbc" src="../assets/anomaly/images/challenge/d2cb2f084d1d45b6a98513a79b944bbc.jpg" data-id="d2cb2f084d1d45b6a98513a79b944bbc.jpg" data-testid="anomaly-modal-image-3"></img>
+                                        </label>
+                                    
+                                </div>
+                            
+                                <div class="anomaly-modal__box" data-index="4">
+                                    
+                                    
+                                        <label class="" for="image-check_5e6714da571b4c8895e1d949f47cedc2" data-testid="anomaly-modal-tile-4">
+                                            <input type="checkbox" class="anomaly-modal__check" name="image-check_5e6714da571b4c8895e1d949f47cedc2" id="image-check_5e6714da571b4c8895e1d949f47cedc2">
+                                            <img class="anomaly-modal__image" alt=" " id="image-5e6714da571b4c8895e1d949f47cedc2" src="../assets/anomaly/images/challenge/5e6714da571b4c8895e1d949f47cedc2.jpg" data-id="5e6714da571b4c8895e1d949f47cedc2.jpg" data-testid="anomaly-modal-image-4"></img>
+                                        </label>
+                                    
+                                </div>
+                            
+                                <div class="anomaly-modal__box" data-index="5">
+                                    
+                                    
+                                        <label class="" for="image-check_23c1e3ae8ce8443183e624800acf2f47" data-testid="anomaly-modal-tile-5">
+                                            <input type="checkbox" class="anomaly-modal__check" name="image-check_23c1e3ae8ce8443183e624800acf2f47" id="image-check_23c1e3ae8ce8443183e624800acf2f47">
+                                            <img class="anomaly-modal__image" alt=" " id="image-23c1e3ae8ce8443183e624800acf2f47" src="../assets/anomaly/images/challenge/23c1e3ae8ce8443183e624800acf2f47.jpg" data-id="23c1e3ae8ce8443183e624800acf2f47.jpg" data-testid="anomaly-modal-image-5"></img>
+                                        </label>
+                                    
+                                </div>
+                            
+                                <div class="anomaly-modal__box" data-index="6">
+                                    
+                                    
+                                        <label class="" for="image-check_ae7275bb00734944820af39a39d39107" data-testid="anomaly-modal-tile-6">
+                                            <input type="checkbox" class="anomaly-modal__check" name="image-check_ae7275bb00734944820af39a39d39107" id="image-check_ae7275bb00734944820af39a39d39107">
+                                            <img class="anomaly-modal__image" alt=" " id="image-ae7275bb00734944820af39a39d39107" src="../assets/anomaly/images/challenge/ae7275bb00734944820af39a39d39107.jpg" data-id="ae7275bb00734944820af39a39d39107.jpg" data-testid="anomaly-modal-image-6"></img>
+                                        </label>
+                                    
+                                </div>
+                            
+                                <div class="anomaly-modal__box" data-index="7">
+                                    
+                                    
+                                        <label class="" for="image-check_d13828125211402eaef44405520c4080" data-testid="anomaly-modal-tile-7">
+                                            <input type="checkbox" class="anomaly-modal__check" name="image-check_d13828125211402eaef44405520c4080" id="image-check_d13828125211402eaef44405520c4080">
+                                            <img class="anomaly-modal__image" alt=" " id="image-d13828125211402eaef44405520c4080" src="../assets/anomaly/images/challenge/d13828125211402eaef44405520c4080.jpg" data-id="d13828125211402eaef44405520c4080.jpg" data-testid="anomaly-modal-image-7"></img>
+                                        </label>
+                                    
+                                </div>
+                            
+                                <div class="anomaly-modal__box" data-index="8">
+                                    
+                                    
+                                        <label class="" for="image-check_aa87423535154d4f8f2a1cfff1de7a8b" data-testid="anomaly-modal-tile-8">
+                                            <input type="checkbox" class="anomaly-modal__check" name="image-check_aa87423535154d4f8f2a1cfff1de7a8b" id="image-check_aa87423535154d4f8f2a1cfff1de7a8b">
+                                            <img class="anomaly-modal__image" alt=" " id="image-aa87423535154d4f8f2a1cfff1de7a8b" src="../assets/anomaly/images/challenge/aa87423535154d4f8f2a1cfff1de7a8b.jpg" data-id="aa87423535154d4f8f2a1cfff1de7a8b.jpg" data-testid="anomaly-modal-image-8"></img>
+                                        </label>
+                                    
+                                </div>
+                            
+                        </div>
+                    </div>
+                    <div class="anomaly-modal__controls">
+                        <button name="challenge-submit" class="btn btn--primary anomaly-modal__submit js-anomaly-modal-submit" form="challenge-form" value="d4cd0dabcf4caa22ad92fab40844c786" >Submit</button>
+                        <p></p>
+                        <input class="feedback-toggle" type="checkbox" id="feedback" unchecked />
+                        <label for="feedback" class="feedback-text">Images not loading?</label>
+                        <div class="feedback-content">
+                            <label class="feedback-instructions">Please email the following code to:</label>
+                            <p class="feedback-instructions">error-lite@duckduckgo.com</p>
+                            <p><label class="feedback-instructions">Code: d4cd0dabcf4caa22ad92fab40844c786</label></p>
+                        </div>
+                    </div>
+            </div>
+        </form>
+        
+
+        
+
+        
+
+        
+
+
+        <br>
+        <p class='html-only'>
+            <!--    This is the Lite version. Get the full-JS version <a href="https://duckduckgo.com/">here</a>. //-->
+        </p>
+    </center>
+    
+        <img src="//duckduckgo.com/t/tqadb?2019127&amp;s=lite&amp;cc=botnet&amp;sc=1&amp;i=-1&amp;p=d72c36ab4cb24f49969ecef4b2b88ad1-a3ec151498ae45aa845c5dda2f14a862-61b69bac6cc64d5886ca11ee9fc0eef2-d2cb2f084d1d45b6a98513a79b944bbc-5e6714da571b4c8895e1d949f47cedc2-23c1e3ae8ce8443183e624800acf2f47-ae7275bb00734944820af39a39d39107-d13828125211402eaef44405520c4080-aa87423535154d4f8f2a1cfff1de7a8b&amp;iadb=0&amp;gk=d4cd0dabcf4caa22ad92fab40844c786&amp;c=dadb&amp;e=0&amp;o=FYGE4bAiNVr8IcPNJsONTSwq441QyPzksuoqxWPyWuuL0OwnOoQUGNKgGm2X8RYI%0A&amp;r=inc"/>
+    
+    
+    
+    
+    
+
+</body>
+
+</html>