Update app.py

app.py

@@ -37,6 +37,44 @@ websites = [
         "https://www.ihg.com"
 ]
 
+BTsites = [
+        "https://www.banyantree.com",
+        "https://www.angsana.com",
+        "https://www.cassia.com",
+        "https://www.dhawa.com",
+        "https://www.garrya.com",
+        "https://www.hommhotels.com",
+        "https://www.foliohotels.com",
+        "https://www.groupbanyan.com",
+        "https://veya.banyantree.com",
+        "https://escape.banyantree.com"
+]
+
+def check_robots(url):
+    try:
+        headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"}        
+        response = requests.head(url, timeout=10)
+        if response.status_code == 200 or response.status_code == 401:
+            return('OK')
+        else:
+            return('Not OK')
+    except requests.exceptions.RequestException as e:
+        print(f"Error checking {url}: {e}")
+        return None
+
+def check_sitemap(url):
+    try:
+        headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"}        
+        new_url = url +"/sitemap.xml"
+        response = requests.head(url, timeout=10)
+        if response.status_code == 200 or response.status_code == 401:
+            return('OK')
+        else:
+            return('Not OK')
+    except requests.exceptions.RequestException as e:
+        print(f"Error checking {url}: {e}")
+        return None
+
 def generate_transparency_links(query,history):
     """
     Generate links to transparency sites for a given query (e.g., brand name or website).
@@ -85,6 +123,26 @@ def runchecks(message,history):
                 results.append([site,speed])
         df_log = pd.DataFrame(results)
         return str(df_log)
+    elseif message == 'check bots' or message == 'Check Bots':
+        for site in BTsites:
+            print(f"Checking robots.txt for {site}...")
+            botsCheck = check_robots(site)
+            time.sleep(5)
+            if botsCheck is not None:
+                print(f"Bots for {site}: {botsCheck} seconds")
+                results.append([site,botsCheck])
+        df_log = pd.DataFrame(results)
+        return str(df_log)
+    elseif message == 'check sitemaps' or message == 'Check Sitemaps':
+        for site in BTsites:
+            print(f"Checking sitemaps for {site}...")
+            mapsCheck = check_sitemap(site)
+            time.sleep(5)
+            if mapsCheck is not None:
+                print(f"Bots for {site}: {mapsCheck} seconds")
+                results.append([site,mapsCheck])
+        df_log = pd.DataFrame(results)
+        return str(df_log)
     else:
         return(generate_transparency_links(message,history))