Update app.py

app.py

@@ -235,36 +235,42 @@ def find_all(purpose,task,history, url, result):
         if url != "" and url != None:    
             out = []
             source = requests.get(url)
-            soup = bs4.BeautifulSoup(source.content,'lxml')
-           
-            rawp=(f'RAW TEXT RETURNED: {soup.text}')
-            cnt=0
-            cnt+=len(rawp)
-            out.append(rawp)
-            out.append("HTML fragments: ")
-            q=("a","p","span","content","article")
-            for p in soup.find_all("a"):
-                out.append([{"LINK TITLE":p.get('title'),"URL":p.get('href'),"STRING":p.string}])
-            c=0
-            out = str(out)
-            rl = len(out)
-            print(f'rl:: {rl}')
-            for i in str(out):
-                if i == " " or i=="," or i=="\n" or i=="/" or i=="." or i=="<":
-                    c +=1
-            print (f'c:: {c}')
-            if c > MAX_HISTORY:
-                print("compressing...")
-                rawp = compress_data(c,purpose,task,out,result)  
+            if source.status_code ==200:
+                soup = bs4.BeautifulSoup(source.content,'lxml')
+               
+                rawp=(f'RAW TEXT RETURNED: {soup.text}')
+                cnt=0
+                cnt+=len(rawp)
+                out.append(rawp)
+                out.append("HTML fragments: ")
+                q=("a","p","span","content","article")
+                for p in soup.find_all("a"):
+                    out.append([{"LINK TITLE":p.get('title'),"URL":p.get('href'),"STRING":p.string}])
+                c=0
+                out = str(out)
+                rl = len(out)
+                print(f'rl:: {rl}')
+                for i in str(out):
+                    if i == " " or i=="," or i=="\n" or i=="/" or i=="." or i=="<":
+                        c +=1
+                print (f'c:: {c}')
+                if c > MAX_HISTORY:
+                    print("compressing...")
+                    rawp = compress_data(c,purpose,task,out,result)  
+                else:
+                    rawp = out 
+                result += rawp
+                    
+                #print (rawp)
+                #print (f'out:: {out}')
+                history += "observation: the search results are:\n {}\n".format(rawp)
+                task = "compile report or complete?"
+                return "MAIN", None, history, task, result
             else:
-                rawp = out 
-            result += rawp
-                
-            #print (rawp)
-            #print (f'out:: {out}')
-            history += "observation: the search results are:\n {}\n".format(rawp)
-            task = "compile report or complete?"
-            return "MAIN", None, history, task, result
+                history += f"observation: That URL string returned an error: {source.status_code}, I should try a different URL string\n"
+                result+="Still Working..."
+                return "MAIN", None, history, task, result
+            
         else: 
             history += "observation: An Error occured\nI need to trigger a search using the following syntax:\naction: SCRAPE_WEBSITE action_input=URL\n"
             return "MAIN", None, history, task, result