Add SEO analysis modules and refactor existing code

- Implemented `SeoOn`, `Seo`, and `WebsiteAndTools` classes for handling on-page SEO, general SEO analysis, and website/tool evaluations.
- Created helper functions for data fetching and response uploading to MongoDB.
- Introduced new page `analyzing_page.py` to manage concurrent analysis of different SEO aspects.
- Refactored `home.py` to integrate new analysis classes and streamline the user interface.
- Added session state management for various SEO metrics and analysis results.
- Enhanced error handling and user feedback during analysis processes.

classes/Off_Page.py

@@ -11,6 +11,7 @@ from helper.button_behaviour import hide_button, unhide_button
 from helper.initialize_analyze_session import initialize_analyze_session
 import pandas as pd
 import asyncio
+import json
 
 class SeoOffPageAnalyst:
     def __init__(self, model_url):
@@ -29,45 +30,20 @@ class SeoOffPageAnalyst:
 
         # AGENT NAME
         #st.header(self.analyst_name)
+        if 'off_page_file_uploaded' not in st.session_state:
+            st.session_state['off_page_file_uploaded'] = ''
     
-    def request_model(self, payload_txt):
-        response = requests.post(self.model_url, json=payload_txt)
+    def request_model(self, payload_txt, headers):
+        response = requests.post(self.model_url, json=payload_txt, headers=headers)
         response.raise_for_status()
         output = response.json()
+        text = output["outputs"][0]["outputs"][0]["results"]["text"]["data"]["text"]
+        text = json.loads(text)
         
-        categories = []
-        remarks = []
-        count = []
+        backlinks = text[0]
+        referring_domains = text[1]
 
-        for key, value in output.items():
-            if key == 'json':
-                for item in value:
-                    categories.append(item.get('elements', 'N/A').replace('_', ' ').title())
-                    remarks.append(item.get('remarks', 'N/A'))
-                    count.append(item.get('count', 'N/A'))
-
-        output = ""
-        for i in range(len(categories)):
-            output += f"\n\n---\n **Category:** {categories[i]}"
-            output += f"\n\n **Remarks:** {remarks[i]}\n\n"
-            output += f"**Count:** {count[i]}"
-        
-        data = {
-            "": [str(category) for category in categories],
-            "Count": [str(count) for count in count],
-            "Remarks": [str(remark) for remark in remarks]
-            
-        }
-
-        df_output = pd.DataFrame(data)
-        '''
-        with st.expander("AI Analysis", expanded=True, icon="🤖"):
-            st.table(df_output.style.set_table_styles(
-                [{'selector': 'th:first-child, td:first-child', 'props': [('width', '20px')]},
-                {'selector': 'th, td', 'props': [('width', '150px'), ('text-align', 'center')]}]
-            ).set_properties(**{'text-align': 'center'}))
-        '''
-        return output
+        return text
     
     def process(self):
          start_time = time.time()
@@ -81,6 +57,7 @@ class SeoOffPageAnalyst:
                             if file_info['type'] == 'pdf':
                                 combined_text += file_info['content'] + "\n"
                             '''
+                        try:
                             if file_info['type'] == 'csv':                                    
                                 # Load CSV
                                 df = pd.read_csv(StringIO(file_info['content'].to_csv(index=True)))
@@ -94,22 +71,29 @@ class SeoOffPageAnalyst:
                                     
                                 combined_text += f"Total Backlinks Count: {num_rows}\n"
                                 combined_text += f"Referring Domain: {unique_domains}"
-                        
+                                st.info("Backlinks - SEMRush Uploaded Successfuly", icon="ℹ️")
+                        except KeyError:
+                            st.info("Incorrect CSV format. Please upload a valid CSV file.")
                         # OUTPUT FOR SEO ANALYST
                         payload_txt = {"question": combined_text}
-                        #result = self.request_model(payload_txt)
+                        headers = {
+                            "Content-Type": "application/json",
+                            "x-api-key": f"{os.getenv('x-api-key')}"
+                        }
+                        #result = self.request_model(payload_txt, headers)
                         
                         #end_time = time.time()
                         #time_lapsed = end_time - start_time
                         
-                        debug_info = {'data_field' : 'Backlinks', 'result': combined_text}
+                        debug_info = {'data_field' : 'Backlinks', 'result': payload_txt}
                         #debug_info = {'url_uuid': self.model_url.split("-")[-1],'time_lapsed' : time_lapsed, 'files': [*st.session_state['uploaded_files']],'payload': payload_txt, 'result': result}
                         collect_telemetry(debug_info)
                         
+                        st.session_state["off_page_file_uploaded"] = 'uploaded'
+                        
                         #with st.expander("Debug information", icon="⚙"):
                         #    st.write(debug_info)
                         st.session_state['analyzing'] = False
-                      
     def row1(self):
             #st.write(self.data_src)
             self.uploaded_files = st.file_uploader('Backlinks - SEMRush', type='csv', accept_multiple_files=True, key="seo_off")

classes/On_Page.py

@@ -0,0 +1,180 @@
+import streamlit as st
+import requests
+from dotenv import load_dotenv
+import os
+import time
+from helper.telemetry import collect_telemetry
+from helper.upload_File import uploadFile
+from helper.button_behaviour import hide_button, unhide_button
+from helper.initialize_analyze_session import initialize_analyze_session
+import pandas as pd
+
+class SeoOn:
+    def __init__(self, model_url):
+        self.uploaded_files = []
+        self.file_dict = {}
+        self.file_gt = {}
+        self.model_url = model_url
+        #self.analyst_name = analyst_name
+        #self.data_src = data_src
+        #self.analyst_description = analyst_description
+        self.initialize()
+        
+        self.row1()
+
+    def initialize(self):
+        # FOR ENV
+        load_dotenv()
+
+        # AGENT NAME
+        #st.header(self.analyst_name)
+
+        # EVALUATION FORM LINK
+        #url = os.getenv('Link')
+        #st.write('Evaluation Form: [Link](%s)' % url)
+
+        # RETURN BUTTON
+        '''try:
+            if st.button("Return", type='primary'):
+                st.switch_page("./pages/home.py")
+        except Exception:
+            pass'''
+        if 'crawl_file' not in st.session_state:
+            st.session_state['crawl_file'] = ''
+        if 'first_meaningful_paint' not in st.session_state:
+            st.session_state['first_meaningful_paint'] = ''
+
+    def request_model(self, payload_txt):
+        response = requests.post(self.model_url, json=payload_txt)
+        response.raise_for_status()
+        output = response.json()
+        
+        categories = []
+        remarks = []
+
+        for key, value in output.items():
+            if key == 'json':
+                for item in value:
+                    categories.append(item.get('elements', 'N/A').replace('_', ' ').title())
+                    remarks.append(item.get('remarks', 'N/A'))
+
+        output = ""
+        for i in range(len(categories)):
+            output += f"\n\n---\n **Category:** {categories[i]}"
+            output += f"\n\n **Remarks:** {remarks[i]}\n\n"
+        
+        data = {
+            "Category": [str(category) for category in categories],
+            "Remarks": [str(footprint) for footprint in remarks],
+        }
+        df_output = pd.DataFrame(data)
+        '''
+        with st.expander("AI Analysis", expanded=True, icon="🤖"):
+            st.table(df_output.style.set_table_styles(
+                [{'selector': 'th:first-child, td:first-child', 'props': [('width', '20px')]},
+                {'selector': 'th, td', 'props': [('width', '150px'), ('text-align', 'center')]}]
+            ).set_properties(**{'text-align': 'center'}))
+        '''
+
+        return output
+    
+    def process(self):
+                session = st.session_state.analyze
+                start_time = time.time()
+                if (self.first_meaningful_paint or self.uploaded_files) and session == 'clicked':
+                    first_meaningful_paint = ""
+                    crawl_file = ""
+                    with st.spinner('SEO On Page Analyst...', show_time=True):
+                        st.write('')
+                        '''
+                        try:
+                            for file_info in st.session_state['uploaded_files'].values():
+                                if file_info['type'] == 'pdf':
+                                    combined_text += file_info['content'] + "\n"
+                                elif file_info['type'] == 'csv':
+                                    try:
+                                        combined_text += "CrawlFile CSV: {"+ file_info['content'].to_csv(index=True) + "\n"
+                                    except AttributeError:
+                                        pass
+                        except KeyError:
+                            pass
+                            
+                        try: 
+                            for f in st.session_state['uploaded_gt'].values():
+                                if f['type'] == 'pdf':
+                                    crawl_file += "GTmetrix: {"+ f['content'] + "}\n"
+                                elif f['type'] == 'csv':
+                                    crawl_file += f['content'].to_csv(index=True) + "\n"
+                        except KeyError:
+                            pass
+                       '''
+                        try:
+                            for file_info in st.session_state['uploaded_files'].values():
+                                if file_info['type'] == 'pdf':
+                                    crawl_file += file_info['content'] + "\n"
+                                elif file_info['type'] == 'csv':
+                                    try:
+                                        crawl_file += "CrawlFile CSV: {"+ file_info['content'].to_csv(index=True) + "\n"
+                                    except AttributeError:
+                                        pass
+                        except KeyError:
+                            pass
+                        try:
+                            first_meaningful_paint += f"\nFirst Meaningful Paint: {self.first_meaningful_paint}"
+                        except KeyError:
+                            pass
+                        
+                        debug_info_first_meaningful_paint = {'data_field' : 'First Meaningful Paint', 'result': first_meaningful_paint}
+                        debug_info_crawl_file = {'data_field' : 'Crawl File', 'result': crawl_file}
+
+                        if self.first_meaningful_paint:
+                            st.session_state['first_meaningful_paint'] = 'uploaded'
+                            collect_telemetry(debug_info_first_meaningful_paint)
+                        if self.uploaded_files:
+                            st.session_state['crawl_file'] = 'uploaded'
+                            collect_telemetry(debug_info_crawl_file)
+                            
+                        #with st.expander("Debug information", icon="⚙"):
+                        #    st.write(debug_info)
+
+                        st.session_state['analyzing'] = False
+                        try:
+                            self.file_dict.popitem()
+                        except KeyError:
+                            pass
+                        
+    def row1(self):
+            #st.write(self.data_src)
+            #self.uploaded_files = st.file_uploader("Upload Backlink List (PDF)", type=['pdf', 'csv'], accept_multiple_files=True, key="seo_on")
+            #self.gtmetrix = st.file_uploader("GTmetrix:", type=['pdf', 'csv'], accept_multiple_files=True, key="seo_on_gt")
+            '''
+            if self.uploaded_files:
+                upload.multiple_upload_file(self.uploaded_files)
+                self.file_dict = upload.file_dict
+            
+            if self.gtmetrix:
+                upload.upload_gt(self.gtmetrix)
+            '''
+            self.uploaded_files = st.file_uploader("Crawl File - ScreamingFrog:", type=['pdf', 'csv'], accept_multiple_files=True)
+            #self.gtmetrix = st.file_uploader("GTmetrix", type=['pdf', 'csv'], accept_multiple_files=True, key="seo_on_gt")
+            if self.uploaded_files:
+                upload.multiple_upload_file(self.uploaded_files)
+                self.file_dict = upload.file_dict
+
+            self.first_meaningful_paint = st.text_input("First Meaningful Paint - GTMetrix:", placeholder='Enter First Meaningful Paint')
+           
+
+            #st.write("") # FOR THE HIDE BUTTON
+            #st.write("") # FOR THE HIDE BUTTON
+            #st.write("AI Analyst Output: ")
+            st.session_state['analyzing'] = False
+            #st.write("") # FOR THE HIDE BUTTON
+            self.process()
+            
+            
+      
+
+if __name__ == "__main__":
+    st.set_page_config(layout="wide")
+
+upload = uploadFile()

classes/On_Page_Crawl.py

@@ -76,9 +76,7 @@ class SeoOnCrawl:
     
     def process(self):
         session = st.session_state.analyze
-        if session == 'clicked':
-                start_time = time.time()
-                if self.uploaded_files:
+        if self.uploaded_files and session == 'clicked':
                     combined_text = ""
                     with st.spinner('SEO On Page Analyst...', show_time=True):
                         st.write('')

classes/Seo.py

@@ -40,7 +40,17 @@ class Seo:
                 st.switch_page("./pages/home.py")
         except Exception:
             pass'''
-    
+        if 'bounce_rate' not in st.session_state:
+            st.session_state['bounce_rate'] = ''
+        if 'page_index' not in st.session_state:
+            st.session_state['page_index'] = ''
+        if 'others' not in st.session_state:
+            st.session_state['others'] = ''
+        if 'df_traffic' not in st.session_state:
+            st.session_state['df_traffic'] = ''
+        if 'df_seo' not in st.session_state:
+            st.session_state['df_seo'] = ''
+           
     def request_model(self, payload_txt):
         response = requests.post(self.model_url, json=payload_txt)
         response.raise_for_status()
@@ -149,9 +159,9 @@ class Seo:
     
     def process (self):
         start_time = time.time()
-        
+
         session = st.session_state.analyze
-        if (self.uploaded_file or self.others or self.uploaded_file_seo) and  session == 'clicked':
+        if ((self.uploaded_file or self.others or self.uploaded_file_seo) or (self.page_index or self.bounce_rate)) and  session == 'clicked':
                     combined_text = ""
                     seo_keywords = ""
                     traffic_channels = ""
@@ -180,8 +190,6 @@ class Seo:
                             direct_traffic = st.session_state['direct_traffic']
                             referral_traffic = st.session_state['referral_traffic']
 
-
-
                             traffic_channels += f"\nOrganic Traffic: {organic_traffic}"
                             traffic_channels += f"\nPaid Traffic: {paid_traffic}"
                             traffic_channels += f"\nDirect Traffic: {direct_traffic}"
@@ -225,7 +233,7 @@ class Seo:
                         payload_txt_bounce_rate = {"question": bounce_rate}
 
 
-                        #result = self.request_model(payload_txt)
+                        #result = self.request_model(payload_txt_seo_keywords)
                         #end_time = time.time()
                         #time_lapsed = end_time - start_time
                         debug_info_seo_keywords = {'data_field' : 'SEO Keywords', 'result': seo_keywords}
@@ -246,14 +254,19 @@ class Seo:
                         }
                         '''
                         if self.bounce_rate:
+                            st.session_state['bounce_rate'] = 'uploaded'
                             collect_telemetry(debug_info_bounce_rate)
                         if self.page_index:
+                            st.session_state['pages_index'] = 'uploaded'
                             collect_telemetry(debug_info_pages_index)
                         if self.others:
+                            st.session_state['others'] = 'uploaded'
                             collect_telemetry(debug_info_traffic_aqcuisition)
                         if self.uploaded_file:
+                            st.session_state['df_traffic'] = 'uploaded'
                             collect_telemetry(debug_info_traffic_channels)
                         if self.uploaded_file_seo:
+                            st.session_state['df_seo'] = 'uploaded'
                             collect_telemetry(debug_info_seo_keywords)
                         
                         #with st.expander("Debug information", icon="⚙"):

classes/Seo_Backlinks.py

@@ -291,12 +291,7 @@ class SeoBacklinks:
                         #with st.expander("Debug information", icon="⚙"):
                         #    st.write(debug_info)
 
-                        for df_traffic in st.session_state.keys():
-                            del st.session_state[df_traffic]
-                        for df_seo in st.session_state.keys():
-                            del st.session_state[df_seo]
-                        for others in st.session_state.keys():
-                            del st.session_state[others]
+
 
                         st.session_state['analyzing'] = False      
 

classes/Social_Media.py

@@ -330,10 +330,6 @@ class SocialMediaAnalyst:
                             with st.expander("Debug information", icon="⚙"):
                                 st.write(debug_info)
 
-                            for df in st.session_state.keys():
-                                del st.session_state[df]
-                            for facebook_ad_campaign in st.session_state.keys():
-                                del st.session_state[facebook_ad_campaign]
 
                             st.session_state['analyzing'] = False 
                     else:

classes/Social_Media_FB.py

@@ -142,10 +142,8 @@ class Facebook:
     def process(self):  
         start_time = time.time()
         session = st.session_state.analyze
-        if session == 'clicked':
-                hide_button()
+        if ((self.fb_organic_post and self.fb_organic_post.name) or (self.fb_ads_campaign and self.fb_ads_campaign.name) or self.facebooks or self.facebook_rr) and session == 'clicked':
                 try:
-                    if (self.fb_organic_post and self.fb_organic_post.name) or (self.fb_ads_campaign and self.fb_ads_campaign.name):
                         combined_text = ""
                         with st.spinner('Social Media Analyst...', show_time=True):
                             st.write('')

classes/response_off.py

@@ -0,0 +1,85 @@
+from io import StringIO
+from urllib.parse import urlparse
+import streamlit as st
+import requests
+from dotenv import load_dotenv
+import os
+import time
+from helper.upload_response import upload_response
+from helper.upload_File import uploadFile
+from helper.button_behaviour import hide_button, unhide_button
+from helper.initialize_analyze_session import initialize_analyze_session
+import pandas as pd
+import asyncio
+import json
+from pymongo import MongoClient
+from helper.data_field import data_field
+
+class SeoOffPageAnalyst:
+    def __init__(self, model_url):
+        self.uploaded_files = []
+        self.file_dict = {}
+        self.model_url = model_url
+        #self.analyst_name = analyst_name
+        #self.data_src = data_src
+        #self.analyst_description = analyst_description
+        self.initialize()
+        self.row1()
+
+    def initialize(self):
+        # FOR ENV
+        load_dotenv()
+
+        # AGENT NAME
+        #st.header(self.analyst_name)
+    def request_model(self, payload_txt, headers):
+        response = requests.post(self.model_url, json=payload_txt, headers=headers)
+        response.raise_for_status()
+        output = response.json()
+        text = output["outputs"][0]["outputs"][0]["results"]["text"]["data"]["text"]
+        text = json.loads(text)
+        
+        backlinks = text[0]
+        referring_domains = text[1]
+
+        return text
+    
+    def process(self):
+         session = st.session_state['off_page_file_uploaded']
+         if session == 'uploaded':
+                    with st.spinner('SEO Off Page Analyst...', show_time=True):
+                        st.write('')
+                        
+                        # OUTPUT FOR SEO ANALYST
+                        payload_txt = {"input_value": data_field("Backlinks"),
+                                       "output_type": "text",
+                                        "input_type": "chat"}
+                        headers = {
+                            "Content-Type": "application/json",
+                            "x-api-key": f"{os.getenv('x-api-key')}"
+                        }
+                        result = self.request_model(payload_txt, headers)
+                        
+                        #end_time = time.time()
+                        #time_lapsed = end_time - start_time
+                        
+                        debug_info = {'data_field' : 'Off Page Analyst', 'result': result}
+                        #debug_info = {'url_uuid': self.model_url.split("-")[-1],'time_lapsed' : time_lapsed, 'files': [*st.session_state['uploaded_files']],'payload': payload_txt, 'result': result}
+                        upload_response(debug_info)
+                        
+                        #with st.expander("Debug information", icon="⚙"):
+                        #    st.write(debug_info)
+                        st.session_state["off_page_file_uploaded"] = ''
+                        st.session_state['analyzing'] = False
+                      
+    def row1(self):
+            st.session_state['analyzing'] = False
+            session = st.session_state['off_page_file_uploaded']
+            if session == 'uploaded':
+                self.process()
+                                       
+
+if __name__ == "__main__":
+    st.set_page_config(layout="wide")
+
+upload = uploadFile()

classes/response_on_page.py

@@ -0,0 +1,139 @@
+import streamlit as st
+import requests
+from dotenv import load_dotenv
+import os
+import time
+from helper.telemetry import collect_telemetry
+from helper.upload_File import uploadFile
+from helper.button_behaviour import hide_button, unhide_button
+from helper.initialize_analyze_session import initialize_analyze_session
+import pandas as pd
+from pymongo import MongoClient
+import json
+from helper.data_field import data_field
+from helper.upload_response import upload_response
+
+class SeoOn:
+    def __init__(self, model_url):
+        self.uploaded_files = []
+        self.file_dict = {}
+        self.file_gt = {}
+        self.model_url = model_url
+        #self.analyst_name = analyst_name
+        #self.data_src = data_src
+        #self.analyst_description = analyst_description
+        self.initialize()
+        
+        self.row1()
+
+    def initialize(self):
+        # FOR ENV
+        load_dotenv()
+
+        # AGENT NAME
+        #st.header(self.analyst_name)
+
+        # EVALUATION FORM LINK
+        #url = os.getenv('Link')
+        #st.write('Evaluation Form: [Link](%s)' % url)
+
+        # RETURN BUTTON
+        '''try:
+            if st.button("Return", type='primary'):
+                st.switch_page("./pages/home.py")
+        except Exception:
+            pass'''
+    
+    def request_model(self, payload_txt, headers):
+        response = requests.post(self.model_url, json=payload_txt, headers=headers)
+        response.raise_for_status()
+        
+        output = response.json()
+        #st.write(output)
+        text = output["outputs"][0]["outputs"][0]["results"]["text"]["data"]["text"]
+        text = json.loads(text)
+        #st.write(text)
+        return text
+    
+    def fetch_data(self, data_field):
+        mongodb_uri = os.getenv("MONGODB_URI")
+        myclient = MongoClient(mongodb_uri)
+        mydb = myclient.get_database()
+        mycol = mydb["df_data"]
+        x = mycol.find_one({"data_field": data_field})
+        x = x["result"]
+        return x
+
+    def process(self):
+                with st.spinner('SEO On Page...', show_time=True):
+                        st.write('')
+                        # OUTPUT FOR SEO ANALYST
+                        headers = {"Content-Type": "application/json", "x-api-key": f"{os.getenv('x-api-key')}"}
+                        payload = ""
+                        try:
+                             payload += self.fetch_data("First Meaningful Paint")
+                        except Exception as e:
+                            pass
+                        try:
+                             payload += self.fetch_data("Crawl File")
+                        except Exception as e:
+                            pass
+
+                        try:
+                            session_first_meaningful_paint = st.session_state['first_meaningful_paint']
+                            session_crawl_file = st.session_state['crawl_file']
+                            if session_first_meaningful_paint or session_crawl_file == 'uploaded':
+                                payload_txt = {"input_value": payload, "output_type": "text", "input_type": "chat"}
+                                payload_txt_model = self.request_model(payload_txt, headers)
+                                debug_info = {'data_field' : 'On Page Analyst', 'result': payload_txt_model}
+                                upload_response(debug_info)
+
+                                st.session_state['first_meaningful_paint'] = ''
+                                st.session_state['crawl_file'] = ''
+                        except Exception as e:
+                             pass
+
+                        #end_time = time.time()
+                        #time_lapsed = end_time - start_time
+
+                        #debug_info = {'data_field' : 'GT Metrix', 'result': result}
+                        
+                        
+                        '''
+                        debug_info = {#'analyst': self.analyst_name,
+                                      'url_uuid': self.model_url.split("-")[-1],
+                                      'time_lapsed' : time_lapsed, 
+                                    'crawl_file': [file.name for file in self.uploaded_files] if self.uploaded_files else ['Not available'],
+                                      'gt_metrix': [file.name for file in self.gtmetrix] if self.gtmetrix else ['Not available'],
+                                      'payload': payload_txt, 
+                                      'result': result}
+                        
+                        if self.gtmetrix:
+                            collect_telemetry(debug_info)
+                        '''
+ 
+                            
+                        #with st.expander("Debug information", icon="⚙"):
+                        #    st.write(debug_info)
+                        
+
+                        st.session_state['analyzing'] = False
+                        try:
+                            self.file_dict.popitem()
+                        except KeyError:
+                            pass
+                        
+    def row1(self):
+           
+            #st.write("") # FOR THE HIDE BUTTON
+            #st.write("") # FOR THE HIDE BUTTON
+            #st.write("AI Analyst Output: ")
+            st.session_state['analyzing'] = False
+            #st.write("") # FOR THE HIDE BUTTON
+            self.process()
+                 
+
+if __name__ == "__main__":
+    st.set_page_config(layout="wide")
+
+upload = uploadFile()

classes/response_seo.py

@@ -0,0 +1,204 @@
+import streamlit as st
+import requests
+from dotenv import load_dotenv
+import os
+import pandas as pd
+import pandas._libs.tslibs.parsing
+import time
+import chardet
+from helper.telemetry import collect_telemetry
+from helper.upload_File import uploadFile
+from helper.button_behaviour import hide_button
+from helper.initialize_analyze_session import initialize_analyze_session
+from pymongo import MongoClient
+from helper.data_field import data_field
+from helper.upload_response import upload_response
+import json
+
+class Seo:
+    def __init__(self, model_url):
+        self.uploaded_files = []
+        self.file_dict = {}
+        self.model_url = model_url
+        #self.analyst_name = analyst_name
+        #self.data_src = data_src
+        #self.analyst_description = analyst_description
+        self.initialize()
+        self.row1()
+
+    def initialize(self):
+        # FOR ENV
+        load_dotenv()
+
+        # AGENT NAME
+        #st.header(self.analyst_name)
+
+        # EVALUATION FORM LINK
+        '''url = os.getenv('Link')
+        st.write('Evaluation Form: [Link](%s)' % url)
+
+        # RETURN BUTTON
+        try:
+            if st.button("Return", type='primary'):
+                st.switch_page("./pages/home.py")
+        except Exception:
+            pass'''
+    
+    def request_model(self, payload_txt, headers):
+        response = requests.post(self.model_url, json=payload_txt, headers=headers)
+        response.raise_for_status()
+        output = response.json()
+        text = output["outputs"][0]["outputs"][0]["results"]["text"]["data"]["text"]
+        text = json.loads(text)
+        #st.write(text)
+        return text
+    
+    def fetch_backlinks(self, data_field):
+        mongodb_uri = os.getenv("MONGODB_URI")
+        myclient = MongoClient(mongodb_uri)
+        mydb = myclient.get_database()
+        mycol = mydb["df_data"]
+        x = mycol.find_one({"data_field": data_field})
+        x = x["result"]['question']
+        return x
+    
+    def fetch_data(self, data_field):
+        mongodb_uri = os.getenv("MONGODB_URI")
+        myclient = MongoClient(mongodb_uri)
+        mydb = myclient.get_database()
+        mycol = mydb["df_data"]
+        x = mycol.find_one({"data_field": data_field})
+        x = x["result"]
+        return x
+    
+    def detect_encoding(self, uploaded_file):
+        result = chardet.detect(uploaded_file.read(100000))
+        uploaded_file.seek(0)  # Reset file pointer to the beginning
+        return result['encoding']
+
+    def keyword_ranking(self, df_seo):
+        keyword_ranking = df_seo
+        st.session_state['keyword_ranking'] = keyword_ranking
+
+        keywords_ranking_sorted = keyword_ranking.sort_values("Position", ascending=True)
+
+        keywords_ranking_top_10 = keywords_ranking_sorted[keywords_ranking_sorted["Position"] <= 10].shape[0]
+        keywords_ranking_top_100 = keywords_ranking_sorted[keywords_ranking_sorted["Position"] <= 100].shape[0]
+
+        keyword_ranking = {
+            'Keyword_top_10': keywords_ranking_top_10,
+            'Keyword_top_100': keywords_ranking_top_100
+        }
+        st.session_state['keyword_ranking'] = keyword_ranking
+
+    def traffic_files(self, df_traffic):
+        traffic_channels = df_traffic
+        try:
+            traffic_channels.rename(columns={traffic_channels.columns[0]: 'date'}, inplace=True)
+            traffic_channels['date'] = pd.to_datetime(traffic_channels['date'], format='mixed')
+        except pandas._libs.tslibs.parsing.DateParseError:
+            pass
+        traffic_channels_sort = traffic_channels.sort_values("date", ascending=False)
+
+        organic_traffic = traffic_channels_sort['Organic Search'].values[0]
+        paid_traffic = traffic_channels_sort['Paid Search'].values[0]
+        direct_traffic = traffic_channels_sort['Direct'].values[0]
+        referral_traffic = traffic_channels_sort['Referral'].values[0]
+
+        st.session_state['organic_traffic'] = organic_traffic
+        st.session_state['paid_traffic'] = paid_traffic
+        st.session_state['direct_traffic'] = direct_traffic
+        st.session_state['referral_traffic'] = referral_traffic
+   
+    def ga4_traffic(self, others):
+        st.session_state['others'] = others
+
+        ga4_paid_social = others['Sessions'].values[0]
+        ga4_organic_traffic = others['Sessions'].values[4]
+        ga4_direct_traffic = others['Sessions'].values[2]
+        ga4_referral_traffic = others['Sessions'].values[3]
+        
+        st.session_state['ga4_paid_social'] = ga4_paid_social
+        st.session_state['ga4_organic_traffic'] = ga4_organic_traffic
+        st.session_state['ga4_direct_traffic'] = ga4_direct_traffic
+        st.session_state['ga4_referral_traffic'] = ga4_referral_traffic
+
+    def delete_sessions(self):
+        try:
+            del st.session_state['df_traffic']
+            del st.session_state['others']
+            del st.session_state['df_seo']
+            del st.session_state['keyword_ranking']   
+            del st.session_state['ga4_paid_social']
+            del st.session_state['ga4_organic_traffic']
+            del st.session_state['ga4_direct_traffic']
+            del st.session_state['ga4_referral_traffic']
+            del st.session_state['organic_traffic']
+            del st.session_state['paid_traffic']
+            del st.session_state['direct_traffic']
+            del st.session_state['referral_traffic']
+        except KeyError:
+            pass
+    
+    def process (self):    
+        with st.spinner('Seo Analyst...', show_time=True):
+                        st.write('')
+                        headers = {"Content-Type": "application/json", "x-api-key": f"{os.getenv('x-api-key')}"}
+                        payload = ""
+                        try:
+                            payload += self.fetch_data("SEO Keywords")
+                        except Exception as e:
+                            pass
+                        try:
+                            payload += self.fetch_data("Traffic Channels")
+                        except Exception as e:
+                            pass
+                        try:
+                            payload += self.fetch_data("Traffic Acquisition")
+                        except Exception as e:
+                            pass
+                        try:
+                            payload += self.fetch_data("Pages Indexed")
+                        except Exception as e:
+                            pass
+                        try:
+                            payload += self.fetch_data("Bounce Rate")
+                        except Exception as e:
+                            pass
+                        try:
+                            payload += self.fetch_backlinks("Backlinks")
+                        except Exception as e:
+                            pass
+
+                        try:
+                            session_bounce_rate = st.session_state['bounce_rate']
+                            session_page_index = st.session_state['pages_index']
+                            session_others = st.session_state['others']
+                            session_traffic_channels = st.session_state['df_traffic']
+                            session_traffic_aqcuisition = st.session_state['df_seo']
+                            if session_bounce_rate or session_page_index or session_others or session_traffic_aqcuisition or session_traffic_channels == 'uploaded':
+                                payload_txt = {"input_value": payload, "output_type": "text", "input_type": "chat"}
+                                payload_txt_model = self.request_model(payload_txt, headers)
+                                debug_info = {'data_field' : 'SEO Analyst', 'result': payload_txt_model}
+                                upload_response(debug_info)
+
+                                st.session_state['bounce_rate'] = ''
+                                st.session_state['pages_index'] = ''
+                                st.session_state['others'] = ''
+                                st.session_state['df_traffic'] = ''
+                                st.session_state['df_seo'] = ''
+                        except Exception as e:
+                            pass
+                        st.session_state['analyzing'] = False    
+             
+    def row1(self):
+            st.session_state['analyzing'] = False
+            #st.write("") # FOR THE HIDE BUTTON
+            #analyze_button = st.button("Analyze", disabled=initialize_analyze_session())
+            
+            self.process()
+
+if __name__ == "__main__":
+    st.set_page_config(layout="wide")
+
+upload = uploadFile()

classes/response_website_and_tools.py

@@ -0,0 +1,232 @@
+import streamlit as st
+import requests
+from dotenv import load_dotenv
+import os
+import time
+from helper.telemetry import collect_telemetry
+from helper.upload_File import uploadFile
+from helper.button_behaviour import hide_button, unhide_button
+from helper.initialize_analyze_session import initialize_analyze_session
+import pandas as pd
+from pymongo import MongoClient
+import json
+from helper.data_field import data_field
+from helper.upload_response import upload_response
+
+class WebsiteAndTools:
+    def __init__(self, model_url):
+        self.uploaded_files = []
+        self.file_dict = {}
+        self.file_gt = {}
+        self.model_url = model_url
+        #self.analyst_name = analyst_name
+        #self.data_src = data_src
+        #self.analyst_description = analyst_description
+        self.initialize()
+        
+        self.row1()
+
+    def initialize(self):
+        # FOR ENV
+        load_dotenv()
+
+        # AGENT NAME
+        #st.header(self.analyst_name)
+
+        # EVALUATION FORM LINK
+        #url = os.getenv('Link')
+        #st.write('Evaluation Form: [Link](%s)' % url)
+
+        # RETURN BUTTON
+        '''try:
+            if st.button("Return", type='primary'):
+                st.switch_page("./pages/home.py")
+        except Exception:
+            pass'''
+    
+    def request_model(self, payload_txt, headers):
+        response = requests.post(self.model_url, json=payload_txt, headers=headers)
+        response.raise_for_status()
+        
+        output = response.json()
+        text = output["outputs"][0]["outputs"][0]["results"]["text"]["data"]["text"]
+        text = json.loads(text)
+        #st.write(text)
+        return text
+    
+    def fetch_data(self, data_field):
+        mongodb_uri = os.getenv("MONGODB_URI")
+        myclient = MongoClient(mongodb_uri)
+        mydb = myclient.get_database()
+        mycol = mydb["df_data"]
+        x = mycol.find_one({"data_field": data_field})
+        x = x["result"]
+        return x
+
+    def process(self):
+                with st.spinner('Website and Tools...', show_time=True):
+                        st.write('')
+                        '''
+                        # OUTPUT FOR WEBSITE RESPONSIVENESS
+                        payload_txt_website_responsiveness = {"result": website_responsiveness}
+                        result_website_responsiveness = self.request_model(payload_txt_website_responsiveness)
+
+                        # OUTPUT FOR CONTENT MANAGEMENT SYSTEM
+                        payload_txt_content_management_system = {"question": content_management_system}
+                        #result_content_management_system = self.request_model(content_management_system)
+
+                        # OUTPUT FOR SSL CERTIFICATE
+                        payload_txt_SSL_certificate = {"question": SSL_certificate}
+                        #result_SSL_certificate = self.request_model(SSL_certificate)
+
+                        # OUTPUT FOR WEB ANALYTICS
+                        payload_txt_web_analytics = {"question": web_analytics}
+                        #result_web_analytics = self.request_model(web_analytics)
+
+                        # OUTPUT FOR CLIENT RELATIONS MANAGEMENT SYSTEM
+                        payload_txt_client_relations_management_system = {"question": client_relations_management_system}
+                        #result_client_relations_management_system = self.request_model(client_relations_management_system)
+                        
+                        # OUTPUT FOR LEAD GENERATION MECHANISM
+                        payload_txt_lead_generation_mechanism = {"question": lead_generation_mechanism}
+                        #result_lead_generation_mechanism = self.request_model(lead_generation_mechanism)
+                        '''
+                        # OUTPUT FOR SEO ANALYST
+                      
+                        #print(x)
+                        headers = {"Content-Type": "application/json", "x-api-key": f"{os.getenv('x-api-key')}"}
+                        try:
+                            payload = ""
+                            session_website_responsiveness = st.session_state['website_responsiveness']
+                            if session_website_responsiveness == 'uploaded':
+                                payload += self.fetch_data("Website Responsiveness")
+                        except Exception as e:
+                             pass
+                        try:
+                            session_content_management_system = st.session_state['content_management_system']
+                            if session_content_management_system == 'uploaded':
+                                payload += self.fetch_data("Content Management System")
+                        except Exception as e:
+                             pass
+                        try:
+                            session_SSL_certificate = st.session_state['SSL_certificate']
+                            if session_SSL_certificate == 'uploaded':
+                                payload += self.fetch_data("SSL Certificate")
+                                
+                        except Exception as e:
+                             pass
+                        try:
+                            session_mobile_responsiveness = st.session_state['mobile_responsiveness']
+                            if session_mobile_responsiveness == 'uploaded':
+                                payload += self.fetch_data("Mobile Responsiveness")
+                        except Exception as e:
+                             pass
+                        try:
+                            session_desktop_loading_speed = st.session_state['desktop_loading_speed']
+                            if session_desktop_loading_speed == 'uploaded':
+                                payload += self.fetch_data("Desktop Loading Speed")
+                        except Exception as e:
+                             pass
+                        try:
+                            session_mobile_loading_speed = st.session_state['mobile_loading_speed']
+                            if session_mobile_loading_speed == 'uploaded':
+                                payload += self.fetch_data("Mobile Loading Speed")
+                        except Exception as e:
+                             pass
+                        try:
+                            session_first_meaningful_paint = st.session_state['first_meaningful_paint']
+                            if session_first_meaningful_paint == 'uploaded':
+                                payload += self.fetch_data("First Meaningful Paint")
+                        except Exception as e:
+                             pass
+                        try:
+                            session_web_analytics = st.session_state['web_analytics']
+                            if session_web_analytics == 'uploaded':
+                                payload += self.fetch_data("Web Analytics")
+                        except Exception as e:
+                             pass
+                        try:
+                            session_client_relations_management_system = st.session_state['client_relations_management_system']
+                            if session_client_relations_management_system == 'uploaded':
+                                payload += self.fetch_data("Client Relations Management System")
+                        except Exception as e:
+                             pass
+                        try:
+                            session_lead_generation_mechanism = st.session_state['lead_generation_mechanism']
+                            if session_lead_generation_mechanism == 'uploaded':
+                                payload += self.fetch_data("Lead Generation Mechanism")
+                        except Exception as e:
+                             pass
+                        
+                        try:
+                            session_website_responsiveness = st.session_state['website_responsiveness']
+                            session_client_relations_management_system = st.session_state['client_relations_management_system']
+                            session_lead_generation_mechanism = st.session_state['lead_generation_mechanism']
+                            session_web_analytics = st.session_state['web_analytics']
+                            session_mobile_responsiveness = st.session_state['mobile_responsiveness']
+                            session_desktop_loading_speed = st.session_state['desktop_loading_speed']
+                            session_mobile_loading_speed = st.session_state['mobile_loading_speed']
+                            session_SSL_certificate = st.session_state['SSL_certificate']
+                            session_content_management_system = st.session_state['content_management_system']
+
+                            if session_website_responsiveness or session_client_relations_management_system or session_lead_generation_mechanism or session_web_analytics or session_mobile_responsiveness or session_desktop_loading_speed or session_mobile_loading_speed or session_content_management_system or session_SSL_certificate == 'uploaded':
+                                payload_txt = {"input_value": payload, "output_type": "text", "input_type": "chat"}
+                                payload_txt_model = self.request_model(payload_txt, headers)
+                                debug_info = {'data_field' : 'Website and Tools Analyst', 'result': payload_txt_model}
+                                upload_response(debug_info)
+                                st.session_state['website_responsiveness'] = ''
+                                st.session_state['content_management_system'] = ''
+                                st.session_state['SSL_certificate'] = ''
+                                st.session_state['mobile_responsiveness'] = ''
+                                st.session_state['desktop_loading_speed'] = ''
+                                st.session_state['mobile_loading_speed'] = ''
+                                st.session_state['first_meaningful_paint'] = ''
+                                st.session_state['web_analytics'] = ''
+                                st.session_state['client_relations_management_system'] = ''
+                                st.session_state['lead_generation_mechanism'] = ''
+                        except Exception as e:
+                             pass
+                        #end_time = time.time()
+                        #time_lapsed = end_time - start_time
+
+                        #debug_info = {'data_field' : 'GT Metrix', 'result': result}
+                        
+                        
+                        '''
+                        debug_info = {#'analyst': self.analyst_name,
+                                      'url_uuid': self.model_url.split("-")[-1],
+                                      'time_lapsed' : time_lapsed, 
+                                    'crawl_file': [file.name for file in self.uploaded_files] if self.uploaded_files else ['Not available'],
+                                      'gt_metrix': [file.name for file in self.gtmetrix] if self.gtmetrix else ['Not available'],
+                                      'payload': payload_txt, 
+                                      'result': result}
+                        
+                        if self.gtmetrix:
+                            collect_telemetry(debug_info)
+                        '''
+ 
+                            
+                        #with st.expander("Debug information", icon="⚙"):
+                        #    st.write(debug_info)
+                        
+
+                        st.session_state['analyzing'] = False
+                        try:
+                            self.file_dict.popitem()
+                        except KeyError:
+                            pass
+                        
+    def row1(self):
+           
+            #st.write("") # FOR THE HIDE BUTTON
+            #st.write("") # FOR THE HIDE BUTTON
+            #st.write("AI Analyst Output: ")
+            st.session_state['analyzing'] = False
+            #st.write("") # FOR THE HIDE BUTTON
+            self.process()
+                 
+
+if __name__ == "__main__":
+    st.set_page_config(layout="wide")
+
+upload = uploadFile()

classes/{On_Page_GT.py → website_and_tools.py}

@@ -9,7 +9,7 @@ from helper.button_behaviour import hide_button, unhide_button
 from helper.initialize_analyze_session import initialize_analyze_session
 import pandas as pd
 
-class SeoOnGT:
+class WebsiteAndTools:
     def __init__(self, model_url):
         self.uploaded_files = []
         self.file_dict = {}
@@ -39,7 +39,26 @@ class SeoOnGT:
                 st.switch_page("./pages/home.py")
         except Exception:
             pass'''
-    
+        if 'website_responsiveness' not in st.session_state:
+            st.session_state['website_responsiveness'] = ''
+        if 'content_management_system' not in st.session_state:
+            st.session_state['content_management_system'] = ''
+        if 'SSL_certificate' not in st.session_state:
+            st.session_state['SSL_certificate'] = ''
+        if 'mobile_responsiveness' not in st.session_state:
+            st.session_state['mobile_responsiveness'] = ''
+        if 'desktop_loading_speed' not in st.session_state:
+            st.session_state['desktop_loading_speed'] = ''
+        if 'mobile_loading_speed' not in st.session_state:
+            st.session_state['mobile_loading_speed'] = ''
+        if 'web_analytics' not in st.session_state:
+            st.session_state['web_analytics'] = ''
+        if 'client_relations_management_system' not in st.session_state:
+            st.session_state['client_relations_management_system'] = ''
+        if 'lead_generation_mechanism' not in st.session_state:
+            st.session_state['lead_generation_mechanism'] = ''
+        
+        
     def request_model(self, payload_txt):
         response = requests.post(self.model_url, json=payload_txt)
         response.raise_for_status()
@@ -77,8 +96,7 @@ class SeoOnGT:
     def process(self):
                 session = st.session_state.analyze
                 start_time = time.time()
-                if (self.website_responsiveness or self.content_management_system or self.SSL_certificate or self.web_analytics or self.client_relations_management_system or self.lead_generation_mechanism or self.first_meaningful_paint or self.mobile_responsiveness or self.mobile_loading_speed or self.desktop_loading_speed) and session == 'clicked':
-                    combined_text = ""
+                if (self.website_responsiveness or self.content_management_system or self.SSL_certificate or self.web_analytics or self.client_relations_management_system or self.lead_generation_mechanism or self.mobile_responsiveness or self.mobile_loading_speed or self.desktop_loading_speed) and session == 'clicked':
                     website_responsiveness = ""
                     content_management_system = ""
                     SSL_certificate = ""
@@ -88,7 +106,6 @@ class SeoOnGT:
                     web_analytics = ""
                     client_relations_management_system = ""
                     mobile_loading_speed = ""
-                    first_meaningful_paint = ""
                     lead_generation_mechanism = ""
                     with st.spinner('SEO On Page Analyst...', show_time=True):
                         st.write('')
@@ -104,28 +121,55 @@ class SeoOnGT:
                                         pass
                         except KeyError:
                             pass
-                        '''
+                            
                         try: 
                             for f in st.session_state['uploaded_gt'].values():
                                 if f['type'] == 'pdf':
-                                    combined_text += "GTmetrix: {"+ f['content'] + "}\n"
+                                    crawl_file += "GTmetrix: {"+ f['content'] + "}\n"
                                 elif f['type'] == 'csv':
-                                    combined_text += f['content'].to_csv(index=True) + "\n"
+                                    crawl_file += f['content'].to_csv(index=True) + "\n"
                         except KeyError:
                             pass
+                        '''
                         try:
                             website_responsiveness += f"\nWebsite Responsiveness: {self.website_responsiveness}"
-                            content_management_system += f"\nContent Management System: {self.content_management_system}%"
+                        except KeyError:
+                            pass
+                        try:
+                            content_management_system += f"\nContent Management System: {self.content_management_system}"
+                        except KeyError:
+                            pass
+                        try:
                             SSL_certificate += f"\nSSL Certificate: {self.SSL_certificate}"
+                        except KeyError:
+                            pass
+                        try:
                             mobile_responsiveness += f"\nMobile Responsiveness: {self.mobile_responsiveness}"
+                        except KeyError:
+                            pass
+                        try:
                             desktop_loading_speed += f"\nDesktop Loading Speed: {self.desktop_loading_speed}"
+                        except KeyError:
+                            pass
+                        try:
                             mobile_loading_speed += f"\nMobile Loading Speed: {self.mobile_loading_speed}"
-                            first_meaningful_paint += f"\nFirst Meaningful Paint: {self.first_meaningful_paint}"
+                        except KeyError:
+                            pass
+                        try:
                             web_analytics += f"\nWeb Analytics: {self.web_analytics}"
+
+                        except KeyError:
+                            pass
+                        try:
                             client_relations_management_system += f"\nClient Relations Management System: {self.client_relations_management_system}"
+                        except KeyError:
+                            pass
+                        try:
                             lead_generation_mechanism += f"\nLead Generation Mechanism: {self.lead_generation_mechanism}"
                         except KeyError:
                             pass
+
+                        
                         # OUTPUT FOR WEBSITE RESPONSIVENESS
                         payload_txt_website_responsiveness = {"question": website_responsiveness}
                         #result_website_responsiveness = self.request_model(website_responsiveness)
@@ -151,19 +195,17 @@ class SeoOnGT:
                         #result_lead_generation_mechanism = self.request_model(lead_generation_mechanism)
 
                         # OUTPUT FOR SEO ANALYST
-                        payload_txt = {"question": combined_text}
+                        #payload_txt = {"question": combined_text}
                         #result = self.request_model(payload_txt)
                         #end_time = time.time()
                         #time_lapsed = end_time - start_time
 
-                        debug_info = {'data_field' : 'GT Metrix', 'result': combined_text}
                         debug_info_website_responsiveness = {'data_field' : 'Website Responsiveness', 'result': website_responsiveness}
                         debug_info_content_management_system = {'data_field' : 'Content Management System', 'result': content_management_system}
                         debug_info_SSL_certificate = {'data_field' : 'SSL Certificate', 'result': SSL_certificate}
                         debug_info_mobile_responsiveness = {'data_field' : 'Mobile Responsiveness', 'result': mobile_responsiveness}
                         debug_info_desktop_loading_speed = {'data_field' : 'Desktop Loading Speed', 'result': desktop_loading_speed}
                         debug_info_mobile_loading_speed = {'data_field' : 'Mobile Loading Speed', 'result': mobile_loading_speed}
-                        debug_info_first_meaningful_paint = {'data_field' : 'First Meaningful Paint', 'result': first_meaningful_paint}
                         debug_info_web_analytics = {'data_field' : 'Web Analytics', 'result': web_analytics}
                         debug_info_client_relations_management_system = {'data_field' : 'Client Relations Management System', 'result': client_relations_management_system}
                         debug_info_lead_generation_mechanism = {'data_field' : 'Lead Generation Mechanism', 'result': lead_generation_mechanism}
@@ -180,24 +222,31 @@ class SeoOnGT:
                             collect_telemetry(debug_info)
                         '''
                         if self.website_responsiveness:
+                            st.session_state['website_responsiveness'] = 'uploaded'
                             collect_telemetry(debug_info_website_responsiveness)
                         if self.content_management_system:
+                            st.session_state['content_management_system'] = 'uploaded'
                             collect_telemetry(debug_info_content_management_system)
                         if self.SSL_certificate:
+                            st.session_state['SSL_certificate'] = 'uploaded'
                             collect_telemetry(debug_info_SSL_certificate)
                         if self.mobile_responsiveness:
+                            st.session_state['mobile_responsiveness'] = 'uploaded'
                             collect_telemetry(debug_info_mobile_responsiveness)
                         if self.desktop_loading_speed:
+                            st.session_state['desktop_loading_speed'] = 'uploaded'
                             collect_telemetry(debug_info_desktop_loading_speed)
                         if self.mobile_loading_speed:
+                            st.session_state['mobile_loading_speed'] = 'uploaded'
                             collect_telemetry(debug_info_mobile_loading_speed)
-                        if self.first_meaningful_paint:
-                            collect_telemetry(debug_info_first_meaningful_paint)
                         if self.web_analytics:
+                            st.session_state['web_analytics'] = 'uploaded'
                             collect_telemetry(debug_info_web_analytics)
                         if self.client_relations_management_system:
+                            st.session_state['client_relations_management_system'] = 'uploaded'
                             collect_telemetry(debug_info_client_relations_management_system)
                         if self.lead_generation_mechanism:
+                            st.session_state['lead_generation_mechanism'] = 'uploaded'
                             collect_telemetry(debug_info_lead_generation_mechanism)
                             
                         #with st.expander("Debug information", icon="⚙"):
@@ -205,9 +254,6 @@ class SeoOnGT:
 
 
                         st.session_state['analyzing'] = False
-
-                        for df_seo in st.session_state.keys():
-                            del st.session_state[df_seo]
                         try:
                             self.file_dict.popitem()
                         except KeyError:
@@ -233,7 +279,6 @@ class SeoOnGT:
             self.mobile_loading_speed = st.text_input("Mobile Loading Speed - GTMetrix:", placeholder='Enter Mobile Loading Speed')
             self.web_analytics = st.text_input("Web Analytics - BuiltWith:", placeholder='Enter Web Analytics')
             self.client_relations_management_system = st.text_input("Client Relations Management System - BuiltWith:", placeholder='Enter Client Relations Management System')
-            self.first_meaningful_paint = st.text_input("First Meaningful Paint - GTMetrix:", placeholder='Enter First Meaningful Paint')
             self.lead_generation_mechanism = st.text_input("Lead Generation Mechanism - Business Context (Lead Generation & Lead Nurturing):", placeholder='Enter Lead Generation Mechanism')
 
             #st.write("") # FOR THE HIDE BUTTON
@@ -242,9 +287,6 @@ class SeoOnGT:
             st.session_state['analyzing'] = False
             #st.write("") # FOR THE HIDE BUTTON
             self.process()
-            
-            
-      
 
 if __name__ == "__main__":
     st.set_page_config(layout="wide")

helper/data_field.py

@@ -0,0 +1,11 @@
+from pymongo import MongoClient
+import os
+def data_field(data_src):
+        mongodb_uri = os.getenv("MONGODB_URI")
+        myclient = MongoClient(mongodb_uri)
+        mydb = myclient.get_database()
+        mycol = mydb["df_data"]
+        x = mycol.find_one({"data_field": data_src})
+        x = x["result"]["question"]
+        #st.write(x)
+        return x

helper/load.py

@@ -0,0 +1,5 @@
+
+def load(num):
+    upload = num
+    return num
+num = 2

helper/upload_button.py

@@ -0,0 +1,25 @@
+import streamlit as st
+
+def hide_button():
+    st.markdown(
+        """
+                    <style>
+                    .element-container:nth-of-type(3) button {
+                    display: none;
+                    }
+                    </style>
+            """,
+            unsafe_allow_html=True,
+        )
+        
+def unhide_button():
+    st.markdown(
+        """
+            <style>
+            element-container:nth-of-type(3) button {
+            display: inline;
+            }
+            </style>
+        """,
+        unsafe_allow_html=True,
+    )

helper/upload_response.py

@@ -0,0 +1,41 @@
+import os
+import json
+from datetime import datetime, timedelta, timezone
+from pymongo import MongoClient
+from dotenv import load_dotenv
+
+load_dotenv()
+
+def upload_response(data):
+    """
+    Sends JSON data to a remote MongoDB instance.
+
+    Args:
+        data (dict): The JSON data to send.
+    """
+    mongodb_uri = os.getenv("MONGODB_URI")
+    if not mongodb_uri:
+        print("Telemetry skipped, no database configured.")
+        return
+
+    try:
+        
+        # Get the current UTC time
+        utc_now = datetime.now(timezone.utc)
+
+        # Define the GMT+8 timezone offset
+        gmt_plus_8 = timezone(timedelta(hours=8))
+
+        # Convert the UTC time to GMT+8
+        timestamp = utc_now.astimezone(gmt_plus_8).isoformat()
+        data['timestamp'] = timestamp
+        
+        client = MongoClient(mongodb_uri)
+        db = client.get_database()  # Use the default database specified in the URI
+        collection = db["df_response"]  # Replace "telemetry" with your desired collection name
+        collection.insert_one(data)
+        print("Data successfully sent to MongoDB.")
+    except Exception as e:
+        print(f"Error sending data to MongoDB: {e}")
+    finally:
+        client.close()

pages/analyzing_page.py

@@ -0,0 +1,87 @@
+import os
+import streamlit as st
+import threading
+from streamlit.runtime.scriptrunner import add_script_run_ctx
+from classes.response_off import SeoOffPageAnalyst
+from classes.response_on_page import SeoOn
+from classes.response_website_and_tools import WebsiteAndTools
+from classes.response_seo import Seo
+
+def run_analysis():
+    # Retrieve uploaded files from session state
+    off_page_file = st.session_state.get('off_page_file_uploaded')
+    gt_file = st.session_state.get('GT_file_uploaded')
+    website_and_tools = st.session_state.get('website_and_tools')
+    seo = st.session_state.get('seo')
+    # Placeholders for status updates
+    off_page_status = st.empty()
+    on_page_status = st.empty()
+    website_and_tools_status = st.empty()
+    seo_status = st.empty()
+    # Function to run SEO Off Page Analysis
+    def run_off_page_analysis():
+        try:
+            off_page_status.info("Starting SEO Off Page Analysis...")
+            result = SeoOffPageAnalyst(os.getenv('MODEL_Off_Page_Analyst'))
+            off_page_status.success("SEO Off Page Analysis completed successfully.")
+            return result
+        except Exception as e:
+            off_page_status.error(f"SEO Off Page Analysis failed: {e}")
+            return None
+
+    # Function to run On Page Analysis
+    def run_on_page_analysis():
+        try:
+            on_page_status.info("Starting On Page Analysis...")
+            result = SeoOn(os.getenv('MODEL_On_Page_Analyst'))
+            on_page_status.success("On Page Analysis completed successfully.")
+            return result
+        except Exception as e:
+            on_page_status.error(f"On Page Analysis failed: {e}")
+            return None
+    
+    def run_website_and_tools_analysis():
+        try:
+            website_and_tools_status.info("Starting Website and Tools Analysis...")
+            result = WebsiteAndTools(os.getenv('Model_Website_and_Tools_Analyst'))
+            website_and_tools_status.success("Website and Tools completed successfully.")
+            return result
+        except Exception as e:
+            on_page_status.error(f"Website and Tools Analysis failed: {e}")
+            return None
+    def run_seo_analysis():
+        try:
+            seo_status.info("Starting SEO Analysis...")
+            result = Seo(os.getenv('MODEL_SEO_Analyst'))
+            seo_status.success("SEO Analysis completed successfully.")
+            return result
+        except Exception as e:
+            seo_status.error(f"SEO Analysis failed: {e}")
+            return None
+        
+    # Create threads for concurrent execution
+    off_page_thread = threading.Thread(target=run_off_page_analysis)
+    on_page_thread = threading.Thread(target=run_on_page_analysis)
+    website_and_tools_thread = threading.Thread(target=run_website_and_tools_analysis)
+    seo_thread = threading.Thread(target=run_seo_analysis)
+
+    # Attach Streamlit context to threads
+    add_script_run_ctx(off_page_thread)
+    add_script_run_ctx(on_page_thread)
+    add_script_run_ctx(website_and_tools_thread)
+    add_script_run_ctx(seo_thread)
+
+    # Start threads
+    off_page_thread.start()
+    on_page_thread.start()
+    website_and_tools_thread.start()
+    seo_thread.start()
+
+    # Wait for threads to complete
+    off_page_thread.join()
+    on_page_thread.join()
+    website_and_tools_thread.join()
+    seo_thread.join()
+
+# Execute the analysis
+run_analysis()

pages/home.py

@@ -1,7 +1,7 @@
 import os
 import streamlit as st
 from classes.Off_Page import SeoOffPageAnalyst
-from classes.On_Page_GT import SeoOnGT
+from classes.On_Page import SeoOn
 from classes.On_Page_Crawl import SeoOnCrawl
 from classes.Seo_Backlinks import SeoBacklinks
 from classes.Seo import Seo
@@ -11,8 +11,10 @@ from classes.Twitter import Twitter
 from classes.Youtube import YouTube
 from classes.Linkedin import Linkedin
 from classes.Tiktok import Tiktok
+from classes.website_and_tools import WebsiteAndTools
 import asyncio
 import time
+from helper.upload_button import hide_button, unhide_button
 
 class DigitalFootprintDashboard:
     def __init__(self):
@@ -31,10 +33,6 @@ class DigitalFootprintDashboard:
             st.session_state['nature'] = ''
         if 'analyze' not in st.session_state:
             st.session_state['analyze'] = ''
-        if 'seo' not in st.session_state:
-            st.session_state['seo'] = ''
-        if 'twitter' not in st.session_state:
-            st.session_state['twitter'] = ''
     
     async def create_row1(self):
         """Create the first row with four columns"""
@@ -48,16 +46,19 @@ class DigitalFootprintDashboard:
             )
             st.session_state.nature = txt
 
-            analyze_button = st.button("Analyze", st.session_state['analyze'])
-            if analyze_button == True:
+            upload_file_button = st.button("Upload File", st.session_state['analyze'])
+            if upload_file_button == True:
                 st.session_state["analyze"] = 'clicked'
-                st.session_state['seo'] = 'clicked'
-                st.session_state['twitter'] = 'clicked'
+                unhide_button()
             else:
                 st.session_state["analyze"] = ''
-                st.session_state['seo'] = ''
-                st.session_state['twitter'] = ''
-            
+
+            analyze_button = st.button("Analyze")
+            if analyze_button == True:
+                st.switch_page("pages/analyzing_page.py")
+            else:
+                hide_button()
+     
         with col2:
             st.write("## Website Traffic")
             self.backlinks = SeoOffPageAnalyst(os.getenv('MODEL_Off_Page_Analyst'))
@@ -85,8 +86,9 @@ class DigitalFootprintDashboard:
 
         with col4:
             st.write("## Website Structure")
-            self.crawl = SeoOnCrawl(os.getenv('MODEL_On_Page_Analyst'))
-            self.gtmetrix = SeoOnGT(os.getenv('MODEL_On_Page_Analyst'))
+            #self.crawl = SeoOnCrawl(os.getenv('MODEL_On_Page_Analyst'))
+            self.on_page = SeoOn(os.getenv('MODEL_On_Page_Analyst'))
+            self.website_and_tools = WebsiteAndTools(os.getenv('MODEL_On_Page_Analyst'))
 
         return col1, col2, col3, col4
 
@@ -101,14 +103,13 @@ class DigitalFootprintDashboard:
             self.youtube.process(), 
             self.linkedin.process(), 
             self.tiktok.process(), 
-            self.crawl.process()
         )
         st.session_state.analyze = False
 
     async def main(self):
         """Main method to run the dashboard"""
         await self.create_row1()
-        await self.run_analysis()
+        #self.run_analysis()
 
 # Main execution
 if __name__ == "__main__":