support browser simulation in google scraping

.gitignore

@@ -4,4 +4,4 @@ __pycache__/
 .gitattributes
 gradio_cached_examples/
 app_*.py
-soup_dump.html
+soup_dump*.html

app.py

@@ -67,7 +67,6 @@ def __countTokens(text):
 st.set_page_config(
     page_title="Mini Perplexity",
     page_icon=C.AI_ICON,
-    # menu_items={"About": None}
 )
 
 
@@ -482,20 +481,8 @@ if prompt := (
             except Exception as e:
                 U.pprint(e)
 
-if "counter" not in st.session_state:
-    st.session_state.counter = 1
-
-st.session_state.counter += 1
-
-import streamlit.components.v1 as components
-components.html(
-    f"<p>{st.session_state.counter}</p>"
-    """
-        <script>
-            console.log("===== script running =====")
-            const input = window.parent.document.querySelector('.stChatInput');
-            console.log({input});
-        </script>
-    """,
-    height=0
-)
+# if st.button("Rerun"):
+#     # __resetButtonState()
+#     st.session_state.chatHistory = []
+#     st.session_state.messages = []
+#     st.rerun()

tools/webScraper.py

@@ -1,48 +1,101 @@
+import os
 from urllib.parse import parse_qs, urlparse
 from bs4 import BeautifulSoup
 import requests
+from typing import TypedDict
+import utils as U
+
+SIMULATE_BROWSER = os.environ.get("SIMULATE_BROWSER_SEARCH") == "true"
+
+
+class SelectorsDict(TypedDict):
+    answer: str
+    answer_desc: str
+    answer_citation: str
+    search_results: str
+    search_results_desc: str
+
+
+SELECTORS: SelectorsDict
+
+if SIMULATE_BROWSER:
+    SELECTORS = {
+        "answer": ".IZ6rdc",
+        "answer_desc": ".LGOjhe",
+        "answer_citation": ".kX21rb.ZYHQ7e",
+        "search_results": ".Ww4FFb",
+        "search_results_desc": ".VwiC3b.yXK7lf",
+    }
+else:
+    SELECTORS = {
+        "answer_desc": "div.PqksIc",
+        "answer_citation": "sub.gMUaMb.r0bn4c.rQMQod",
+        "search_results": "div.egMi0.kCrYT",
+        "search_results_desc": "div.BNeawe.s3v9rd.AP7Wnd .BNeawe.s3v9rd.AP7Wnd:last-child",
+    }
 
 
 def scrapeGoogleSearch(query):
+    U.pprint(f"{SIMULATE_BROWSER=}")
     finalResponse = []
 
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.5",
+        "Referer": "https://www.google.com/",
+        "DNT": "1",  # Do Not Track Request Header
+        "Connection": "keep-alive",
+        "Upgrade-Insecure-Requests": "1"
+    }
+
     searchUrl = f"https://www.google.com/search?q={query}"
-    response = requests.get(searchUrl)
-    if response.status_code == 200:
-        soup = BeautifulSoup(response.text, 'html.parser')
-        with open('soup_dump.html', 'w', encoding='utf-8') as file:
-            file.write(soup.prettify())
-
-        results = soup.find('body')
-        mainDiv = soup.find('div', attrs={'id': 'main'})
-        answerDiv = (
-            mainDiv.select_one('div.PqksIc')
-            or mainDiv.select_one('div.BNeawe.iBp4i')
-        )
-        if answerDiv:
-            citationDateDiv = answerDiv.select_one('sub.gMUaMb.r0bn4c.rQMQod')
-            citationDate = citationDateDiv.text if citationDateDiv else ""
-            answerText = answerDiv.text.replace(citationDate, '').strip()
-            citationText = f"Citation Date: {citationDate}" if citationDate else ""
-            finalResponse.append(f"Verified Answer:\n{answerText}\n{citationText}\n\n\n")
-
-        results = mainDiv.select('div.egMi0.kCrYT')
-        resultsDesc = mainDiv.select('div.BNeawe.s3v9rd.AP7Wnd .BNeawe.s3v9rd.AP7Wnd:last-child')
-
-        if results:
-            finalResponse.append("Search Results:\n")
-
-        for (i, result) in enumerate(results[:10]):
-            title = result.find('h3').text
-            link = result.find('a')['href']
-            parsedUrl = urlparse(link)
-            urlParams = parse_qs(parsedUrl.query)
-            link = urlParams.get('q', [None])[0]
-            desc = resultsDesc[i].text
-            finalResponse.append(f"Title: {title}")
-            finalResponse.append(f"Description: {desc}")
-            finalResponse.append(f"Link: {link}\n")
-    else:
-        print("Failed to retrieve search results.")
+
+    # Use a session to maintain cookies
+    with requests.Session() as session:
+        if SIMULATE_BROWSER:
+            session.headers.update(headers)
+        response = session.get(searchUrl)
+        if response.status_code == 200:
+            soup = BeautifulSoup(response.text, "html.parser")
+            with open("soup_dump.html", "w", encoding="utf-8") as file:
+                file.write(soup.prettify())
+
+            results = soup.find("body")
+            mainDiv = soup.find("div", attrs={"id": "main"})
+            answerText = ""
+            if SELECTORS.get("answer"):
+                mainAnswerDiv = mainDiv.select_one(SELECTORS["answer"])
+                if mainAnswerDiv:
+                    mainAnswer = mainAnswerDiv.text.strip()
+                    answerText = f"**{mainAnswer}**. "
+
+            answerDescDiv = mainDiv.select_one(SELECTORS["answer_desc"])
+            if answerDescDiv:
+                citationDateDiv = answerDescDiv.select_one(SELECTORS["answer_citation"])
+                citationDate = citationDateDiv.text if citationDateDiv else ""
+                answerText += answerDescDiv.text.replace(citationDate, "").strip()
+                citationText = f"Citation Date: {citationDate}" if citationDate else ""
+                finalResponse.append(f"Verified Answer:\n{answerText}\n{citationText}\n\n\n")
+
+            results = mainDiv.select(SELECTORS["search_results"])
+            resultsDesc = mainDiv.select(SELECTORS["search_results_desc"])
+
+            if results:
+                finalResponse.append("Search Results:\n")
+
+            for (i, result) in enumerate(results[:10]):
+                title = result.find("h3").text
+                link = result.find("a")["href"]
+                if not SIMULATE_BROWSER:
+                    parsedUrl = urlparse(link)
+                    urlParams = parse_qs(parsedUrl.query)
+                    link = urlParams.get("q", [None])[0]
+                desc = resultsDesc[i].text
+                finalResponse.append(f"Title: {title}")
+                finalResponse.append(f"Description: {desc}")
+                finalResponse.append(f"URL: {link}\n")
+        else:
+            print("Failed to retrieve search results.")
 
     return "\n".join(finalResponse)

utils.py

@@ -15,6 +15,10 @@ def applyCommonStyles():
             font-family: 'Raleway';
         }
 
+        .stButton p {
+            font-size: 0.9rem;
+        }
+
         @keyframes blinker {
             0% {
                 opacity: 1;