Add WebsiteAudienceAcquisition class and integrate into analysis workflow; refactor upload handling in uploadFile

classes/Off_Page.py

@@ -3,14 +3,12 @@ from urllib.parse import urlparse
 import streamlit as st
 import requests
 from dotenv import load_dotenv
-import os
 import time
 from helper.telemetry import collect_telemetry
 from helper.upload_File import uploadFile
 from helper.button_behaviour import hide_button, unhide_button
 from helper.initialize_analyze_session import initialize_analyze_session
 import pandas as pd
-import asyncio
 import json
 
 class SeoOffPageAnalyst:
@@ -32,6 +30,10 @@ class SeoOffPageAnalyst:
         #st.header(self.analyst_name)
         if 'off_page_file_uploaded' not in st.session_state:
             st.session_state['off_page_file_uploaded'] = ''
+        if 'website_audience' not in st.session_state:
+            st.session_state['website_audience'] = ''
+        if 'uploaded_files' not in st.session_state:
+            st.session_state['uploaded_files'] = ''
     
     def request_model(self, payload_txt, headers):
         response = requests.post(self.model_url, json=payload_txt, headers=headers)
@@ -48,32 +50,64 @@ class SeoOffPageAnalyst:
     def process(self):
          start_time = time.time()
          session = st.session_state['analyze']
-         if self.uploaded_files and session == 'clicked':
+         if (self.uploaded_files or self.website_audience) and session == 'clicked':
                     combined_text = ""
+                    website_audience = ""
                     with st.spinner('SEO Off Page Analyst...', show_time=True):
                         st.write('')
-                        for file_info in st.session_state['uploaded_files'].values():
-                            '''
-                            if file_info['type'] == 'pdf':
-                                combined_text += file_info['content'] + "\n"
-                            '''
                         try:
-                            if file_info['type'] == 'csv':                                    
-                                # Load CSV
-                                df = pd.read_csv(StringIO(file_info['content'].to_csv(index=True)))
-                                
-                                # Count total rows
-                                num_rows = len(df)
-                                
-                                # Extract unique domains from 'Source url'
-                                df['Source Domain'] = df['Source url'].apply(lambda x: urlparse(x).netloc)
-                                unique_domains = df['Source Domain'].nunique()
+                            for file_info in st.session_state['uploaded_files'].values():
+                                '''
+                                if file_info['type'] == 'pdf':
+                                    combined_text += file_info['content'] + "\n"
+                                '''
+                            try:
+                                if file_info['type'] == 'csv':                                    
+                                    # Load CSV
+                                    df = pd.read_csv(StringIO(file_info['content'].to_csv(index=True)))
                                     
-                                combined_text += f"Total Backlinks Count: {num_rows}\n"
-                                combined_text += f"Referring Domain: {unique_domains}"
-                                st.info("Backlinks - SEMRush Uploaded Successfuly", icon="ℹ️")
+                                    # Count total rows
+                                    num_rows = len(df)
+                                    
+                                    # Extract unique domains from 'Source url'
+                                    df['Source Domain'] = df['Source url'].apply(lambda x: urlparse(x).netloc)
+                                    unique_domains = df['Source Domain'].nunique()
+                                        
+                                    combined_text += f"Total Backlinks Count: {num_rows}\n"
+                                    combined_text += f"Referring Domain: {unique_domains}"
+                                    st.info("Backlinks - SEMRush Uploaded Successfuly", icon="ℹ️")
+                            except KeyError:
+                                st.info("Incorrect CSV format. Please upload a valid CSV file.")
+                            except UnboundLocalError:
+                                 pass
+                        except AttributeError:
+                                 pass
                         except KeyError:
-                            st.info("Incorrect CSV format. Please upload a valid CSV file.")
+                             pass
+                        
+                        try:
+                            # Check if upload_website_audience exists in session state and is a dictionary
+                            if 'upload_website_audience' in st.session_state and isinstance(st.session_state['upload_website_audience'], dict):
+                                for file_name, file_info in st.session_state['upload_website_audience'].items():
+                                    try:
+                                        if file_info['type'] == 'csv':
+                                            # Since file_info['content'] is already a DataFrame (from your earlier code)
+                                            # No need to convert back from string to DataFrame
+                                            df = file_info['content']
+                                            
+                                            # Process your DataFrame here
+                                            # Instead of reading from StringIO, just use the DataFrame directly
+                                            website_audience += f"Website Audience Acquisition {df}\n"
+                                            
+                                            st.info("Website Audience Acquisition Uploaded Successfully", icon="ℹ️")
+                                    except KeyError:
+                                        st.info(f"Incorrect format for {file_name}. Please upload a valid CSV file.")
+                            else:
+                                st.info("No website audience data available. Please upload CSV files first.")
+                        except Exception as e:
+                            st.error(f"Error processing data: {str(e)}")
+                        
+
                         # OUTPUT FOR SEO ANALYST
 
                         #result = self.request_model(payload_txt, headers)
@@ -82,19 +116,31 @@ class SeoOffPageAnalyst:
                         #time_lapsed = end_time - start_time
                         
                         debug_info = {'data_field' : 'Backlinks', 'result': combined_text}
+                        debug_info_website_audience = {'data_field' : 'Website Audience Acquisition', 'result': website_audience}
                         #debug_info = {'url_uuid': self.model_url.split("-")[-1],'time_lapsed' : time_lapsed, 'files': [*st.session_state['uploaded_files']],'payload': payload_txt, 'result': result}
-                        collect_telemetry(debug_info)
                         
-                        st.session_state["off_page_file_uploaded"] = 'uploaded'
+                        if self.uploaded_files:
+                            st.session_state['off_page_file_uploaded'] = 'uploaded'
+                            collect_telemetry(debug_info)
+                        if self.website_audience:
+                            st.session_state['website_audience'] = 'uploaded'
+                            collect_telemetry(debug_info_website_audience)
+
                         
                         #with st.expander("Debug information", icon="⚙"):
                         #    st.write(debug_info)
                         st.session_state['analyzing'] = False
+    
     def row1(self):
             #st.write(self.data_src)
             self.uploaded_files = st.file_uploader('Backlinks - SEMRush', type='csv', accept_multiple_files=True, key="seo_off")
+            self.website_audience = st.file_uploader('Website Audience Acquisition - GA4', type='csv', accept_multiple_files=True, key="website_audiences")
+            #self.website_audience = st.text_input("Website Audience Acquisition:", placeholder='Enter Website Audience Acquisition')
+
             if self.uploaded_files:
                 upload.multiple_upload_file(self.uploaded_files)
+            if self.website_audience:
+                 upload.upload_website_audience(self.website_audience)
                 
             #st.write("") # FOR THE HIDE BUTTON
             #st.write("") # FOR THE HIDE BUTTON

classes/response_conversion_analyst.py

@@ -28,7 +28,6 @@ class ConversionAnalyst:
     def request_model(self, payload_txt, headers):
         response = requests.post(self.model_url, json=payload_txt, headers=headers)
         response.raise_for_status()
-        print(response)
         output = response.json()
         #st.write(output)
         text = output["outputs"][0]["outputs"][0]["results"]["text"]["data"]["text"]

classes/response_desired_outcome.py

@@ -28,7 +28,6 @@ class DesiredOutcome:
     def request_model(self, payload_txt, headers):
         response = requests.post(self.model_url, json=payload_txt, headers=headers)
         response.raise_for_status()
-        print(response)
         output = response.json()
         #st.write(output)
         text = output["outputs"][0]["outputs"][0]["results"]["text"]["data"]["text"]

classes/response_website_audience_acquisition.py

@@ -0,0 +1,91 @@
+import streamlit as st
+import requests
+from dotenv import load_dotenv
+import os
+from helper.upload_response import upload_response
+from helper.upload_File import uploadFile
+from pymongo import MongoClient
+import json
+
+class WebsiteAudienceAcquisition:
+    def __init__(self, model_url):
+        self.uploaded_files = []
+        self.file_dict = {}
+        self.model_url = model_url
+        #self.analyst_name = analyst_name
+        #self.data_src = data_src
+        #self.analyst_description = analyst_description
+        self.initialize()
+        self.row1()
+
+    def initialize(self):
+        # FOR ENV
+        load_dotenv()
+
+        # AGENT NAME
+        #st.header(self.analyst_name)
+    
+    def request_model(self, payload_txt, headers):
+        response = requests.post(self.model_url, json=payload_txt, headers=headers)
+        response.raise_for_status()
+        output = response.json()
+        #st.write(output)
+        text = output["outputs"][0]["outputs"][0]["results"]["text"]["data"]["text"]
+        #text = json.loads(text)
+        #st.write(text)
+        return text
+    
+    def fetch_data(self, data_field):
+        mongodb_uri = os.getenv("MONGODB_URI")
+        myclient = MongoClient(mongodb_uri)
+        mydb = myclient.get_database()
+        mycol = mydb["df_data"]
+        
+        # Sort by timestamp field in descending order
+        x = mycol.find_one(
+            {"data_field": data_field},
+            sort=[("timestamp", -1)]  
+        )
+        
+        x = x["result"]
+        return x
+    
+    def process(self):
+                with st.spinner('Website Audience Acquisition...', show_time=True):
+                        st.write('')
+                        headers = {"Content-Type": "application/json", "x-api-key": f"{os.getenv('x-api-key')}"}         
+                        try:
+                                payload_txt = {"input_value": self.payload, "output_type": "text", "input_type": "chat"}
+                                payload_txt_model = self.request_model(payload_txt, headers)
+                                debug_info = {'data_field' : 'Website Audience Acquisition', 'result': payload_txt_model}
+                                upload_response(debug_info)
+                                st.session_state['website_audience'] = ''
+
+                                count = 0
+                        except Exception as e:
+                            pass
+                        st.session_state['analyzing'] = False    
+                      
+    def row1(self):
+            st.session_state['analyzing'] = False
+            self.payload = ""  
+            count = 0
+            try:
+                session_content_outside_the_website = st.session_state['website_audience']
+                if session_content_outside_the_website == 'uploaded':
+                    count += 1
+                    self.payload += self.fetch_data("Website Audience Acquisition")
+            except Exception as e:
+                pass
+            
+            if count >= 1:
+                name = self.fetch_data("Client Name")
+                website = self.fetch_data("Client Website")
+                self.payload = name + website + self.payload
+                self.process()
+                                       
+
+if __name__ == "__main__":
+    st.set_page_config(layout="wide")
+
+upload = uploadFile()

helper/upload_File.py

@@ -28,6 +28,48 @@ class uploadFile:
                     pass
    
             st.session_state['uploaded_files'] = self.file_dict
+    
+    def upload_website_audience(self, uploaded_files):
+        for _ in range(len(self.file_dict)):
+            self.file_dict.popitem() 
+
+        for uploaded_file in uploaded_files:
+            if uploaded_file.type == "application/pdf":
+                try:
+                    with pymupdf.open(stream=uploaded_file.read(), filetype="pdf") as doc:  
+                        text = chr(12).join([page.get_text() for page in doc])
+                        self.file_dict[uploaded_file.name] = {'type': 'pdf', 'content': text}
+                except Exception:
+                    pass
+            elif uploaded_file.type == "text/csv":
+                try:
+                    # Skip comment lines that start with #
+                    df = pd.read_csv(
+                        uploaded_file,
+                        comment='#',  # Treat lines starting with # as comments
+                        engine='python'  # Use more flexible engine
+                    )
+                    self.file_dict[uploaded_file.name] = {'type': 'csv', 'content': df}
+                except Exception as e:
+                    print(f"Error processing CSV: {str(e)}")
+                    # If that fails, you could try a more manual approach
+                    try:
+                        uploaded_file.seek(0)
+                        raw_text = uploaded_file.read().decode('utf-8')
+                        # Get only non-comment lines
+                        data_lines = [line for line in raw_text.split('\n') if not line.strip().startswith('#')]
+                        
+                        # Use StringIO to create a file-like object from the filtered lines
+                        from io import StringIO
+                        csv_data = StringIO('\n'.join(data_lines))
+                        
+                        # Read from the filtered data
+                        df = pd.read_csv(csv_data)
+                        self.file_dict[uploaded_file.name] = {'type': 'csv', 'content': df}
+                    except Exception as e:
+                        print(f"Second attempt failed: {str(e)}")
+            
+            st.session_state['upload_website_audience'] = self.file_dict
 
     def upload_file_seo(self, uploaded_files):
         for _ in range(len(self.file_dict)):

pages/analyzing_page.py

@@ -17,6 +17,7 @@ from classes.response_target_market import TargetMarket
 from classes.response_df_overview import dfOverview
 from classes.response_desired_outcome import DesiredOutcome
 from classes.response_conversion_analyst import ConversionAnalyst
+from classes.response_website_audience_acquisition import WebsiteAudienceAcquisition
 from classes.response_executive_summary import ExecutiveSummary
 from classes.response_snapshot import Snapshot
 
@@ -72,10 +73,10 @@ def run_analysis():
         "df_overview": st.empty(),
         "desired_outcome": st.empty(),
         "conversion": st.empty(),
+        "website_audience": st.empty(),
         "snapshot": st.empty(),
         "executive_summary": st.empty(),
         
-        
     }
     
     # Create thread-safe handlers for each analysis type
@@ -237,6 +238,17 @@ def run_analysis():
             handler.update_error(f"Conversion Analysis failed: {str(e)}")
             return None
             
+    def run_website_audience():
+        handler = handlers["website_audience"]
+        try:
+            handler.update_info("Running Website Audience Acquisition Analysis...")
+            result = WebsiteAudienceAcquisition(os.getenv('Model_Website_Audience_Acquisition_Analyst'))
+            handler.update_success("Website Audience Acquisition Analysis completed successfully.")
+            return result
+        except Exception as e:
+            handler.update_error(f"Website Audience Acquisition Analysis failed: {str(e)}")
+            return None
+        
     def run_snapshot_analysis():
         handler = handlers["snapshot"]
         try:
@@ -274,7 +286,8 @@ def run_analysis():
         (run_target_market_analysis, "target_market"),
         (run_df_overview_analysis, "df_overview"),
         (run_desired_outcomes_analysis, "desired_outcome"),
-        (run_conversion_analysis, "conversion")
+        (run_conversion_analysis, "conversion"),
+        (run_website_audience, "website_audience")
     ]
     
     # Create and start first batch threads with small delays to prevent UI conflicts

pages/home.py

@@ -65,8 +65,7 @@ class DigitalFootprintDashboard:
             
             self.client_summary = CientSummary()
             
-            
-            
+               
         with col2:
             st.write("## Website Traffic")
             self.backlinks = SeoOffPageAnalyst(os.getenv('MODEL_Off_Page_Analyst'))