new search query generator, removes js render in zen rows

app.py

@@ -5,8 +5,8 @@ from time import asctime
 import gradio as gr
 from llama_index.core import Document, VectorStoreIndex
 
-from generate_response import generate_chat_response_with_history, set_llm, is_search_query, condense_question, \
-    generate_chat_response_with_history_rag_return_response
+from generate_response import generate_chat_response_with_history, set_llm, is_search_query, \
+    generate_chat_response_with_history_rag_return_response, google_question
 from read_write_index import read_write_index
 from web_search import search
 
@@ -17,12 +17,13 @@ rag_similarity = False
 
 
 def google_search_chat(message, history):
-    condensed_question = condense_question(message, history)
-    if is_search_query(condensed_question):
-        search_results = search(message, condensed_question)
+    gquestion = google_question(message, history)
+    if is_search_query(gquestion):
+        search_results = search(message, gquestion)
         print(f'Search results returned: {len(search_results)}')
+
         relevant_content = ""
-        for index, result in enumerate(search_results):
+        for result in enumerate(search_results):
             relevant_content = relevant_content + "\n" + ''.join(result['html'])
 
         if relevant_content != "":

generate_response.py

@@ -152,11 +152,11 @@ def collect_history(message, history):
     return messages
 
 
-def condense_question(message, history):
+def google_question(message, history):
     DEFAULT_TEMPLATE = """\
         Given a conversation (between Human and Assistant) and a follow up message from Human, \
-        rewrite the message to be a standalone question that captures all relevant context \
-        from the conversation.
+        rewrite the message to be a standalone web engine search query that captures all relevant context \
+        from the conversation in keywords.
         
         <Chat History>
         {chat_history}

web_search.py

@@ -18,18 +18,12 @@ logger = logging.getLogger("agent_logger")
 
 
 def search(msg, query_phrase):
+    google_text = ""
     try:
-        # this call extracts keywords from the statement and rewrites it into a better search phrase with gpt3.5
-        # query_phrase, keywords = ut.get_search_phrase_and_keywords(msg, [])
-        google_text = ""
-        try:
-            print(f"asking google {msg}; rephrased: {query_phrase}")
-            google_text = search_google(msg, query_phrase)
-        except:
-            traceback.print_exc()
-
+        print(f"asking google {msg}; rephrased: {query_phrase}")
+        google_text = search_google(msg, query_phrase)
+        print(f"Found {len(google_text)} items")
         print("\n\nFinal response: ")
-
         for item in google_text:
             print(
                 f"\n##############################################################################################\nSource: {item['source']}"
@@ -42,7 +36,7 @@ def search(msg, query_phrase):
         raise KeyboardInterrupt
     except:
         traceback.print_exc()
-    return ""
+    return google_text
 
 
 # Define a function to make a single URL request and process the response
@@ -54,10 +48,10 @@ def process_url(url):
             warnings.simplefilter("ignore")
             try:
                 client = ZenRowsClient(os.getenv('zenrows_api_key'))
-                params = {"js_render": "true",
-                          "js_instructions": "%5B%7B%22click%22%3A%22.selector%22%7D%2C%7B%22wait%22%3A500%7D%2C%7B%22fill%22%3A%5B%22.input%22%2C%22value%22%5D%7D%2C%7B%22wait_for%22%3A%22.slow_selector%22%7D%5D",
-                          }
-                response = client.get(url, params=params)
+                # params = {"js_render": "true", "json_response": "true",
+                #           "js_instructions": "%5B%7B%22click%22%3A%22.selector%22%7D%2C%7B%22wait%22%3A500%7D%2C%7B%22fill%22%3A%5B%22.input%22%2C%22value%22%5D%7D%2C%7B%22wait_for%22%3A%22.slow_selector%22%7D%5D",
+                #           }
+                response = client.get(url)
                 print(f'got response, status: {response.status_code}')
                 result = response.text
                 if len(result) > 0: