from crewai_tools import BaseTool import requests from bs4 import BeautifulSoup from selenium import webdriver from selenium.webdriver.chrome.options import Options import time from crewai import Agent, Task, Crew, LLM, Process from crewai_tools import ScrapeWebsiteTool from crewai_tools import EXASearchTool from crewai_tools import ScrapeWebsiteTool from pydantic import BaseModel import json import re import streamlit as st import os class WebsiteAnalyzerTool(BaseTool): name: str = "WebsiteAnalyzer" description: str = "A tool for comprehensive website analysis, including UX assessment, loading speed, mobile responsiveness, content quality, CTA effectiveness, and SEO strategies." def _run(self, url: str) -> str: analyzer = WebsiteAnalyzer(url) results = analyzer.run_analysis() return self.format_results(results) def format_results(self, results): formatted = "Website Analysis Results:\n\n" for key, value in results.items(): formatted += f"{key}:\n{value}\n\n" return formatted class WebsiteAnalyzer: def __init__(self, url): self.url = url self.soup = None self.driver = None def fetch_page(self): response = requests.get(self.url) self.soup = BeautifulSoup(response.content, 'html.parser') def setup_selenium(self): chrome_options = Options() chrome_options.add_argument("--headless") self.driver = webdriver.Chrome(options=chrome_options) def analyze_ux(self): ux_score = 0 if self.soup.find('nav'): ux_score += 1 if self.soup.find('footer'): ux_score += 1 if self.soup.find('form'): ux_score += 1 return f"UX Score: {ux_score}/3" def analyze_loading_speed(self): start_time = time.time() requests.get(self.url) load_time = time.time() - start_time return f"Loading Speed: {load_time:.2f} seconds" def analyze_mobile_responsiveness(self): self.setup_selenium() self.driver.get(self.url) self.driver.set_window_size(375, 812) # iPhone X dimensions viewport = self.driver.find_element("css selector", "meta[name='viewport']") has_viewport = viewport is not None scroll_width = self.driver.execute_script("return document.body.scrollWidth") client_width = self.driver.execute_script("return document.documentElement.clientWidth") no_horizontal_scroll = scroll_width <= client_width self.driver.quit() return f"Mobile Responsive: {'Yes' if has_viewport and no_horizontal_scroll else 'No'}" def analyze_content_quality(self): paragraphs = self.soup.find_all('p') word_count = sum(len(p.text.split()) for p in paragraphs) headings = self.soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6']) return f"Word Count: {word_count}, Number of Headings: {len(headings)}" def analyze_cta(self): buttons = self.soup.find_all('button') links = self.soup.find_all('a', class_=['btn', 'button', 'cta']) cta_count = len(buttons) + len(links) return f"Number of potential CTAs: {cta_count}" def analyze_seo(self): title = self.soup.title.string if self.soup.title else "No title" meta_description = self.soup.find('meta', attrs={'name': 'description'}) meta_description = meta_description['content'] if meta_description else "No meta description" h1_tags = len(self.soup.find_all('h1')) return f"Title: {title}\nMeta Description: {meta_description}\nNumber of H1 tags: {h1_tags}" def analyze_social_media(self): social_media = { "Facebook": None, "Twitter": None, "LinkedIn": None, "Instagram": None, "YouTube": None } # Look for common patterns for social media links for a_tag in self.soup.find_all('a', href=True): href = a_tag['href'].lower() if "facebook.com" in href: social_media["Facebook"] = a_tag['href'] elif "twitter.com" in href or "x.com" in href: social_media["Twitter"] = a_tag['href'] elif "linkedin.com" in href: social_media["LinkedIn"] = a_tag['href'] elif "instagram.com" in href: social_media["Instagram"] = a_tag['href'] elif "youtube.com" in href: social_media["YouTube"] = a_tag['href'] # Format the output to include the links formatted_social_media = [f"{platform}: {link}" for platform, link in social_media.items() if link] return f"Social Media Presence:\n" + "\n".join(formatted_social_media) if formatted_social_media else "Social Media Presence: None" def suggest_improvements(self): suggestions = [] if len(self.soup.find_all('img', alt="")) > 0: suggestions.append("Add alt text to all images for better accessibility and SEO.") if not self.soup.find('meta', attrs={'name': 'description'}): suggestions.append("Add a meta description for better search engine results.") if len(self.soup.find_all('a', href="#")) > 0: suggestions.append("Replace empty links with meaningful ones or remove them.") return "\n".join(suggestions) def run_analysis(self): self.fetch_page() results = { "UX Assessment": self.analyze_ux(), "Loading Speed": self.analyze_loading_speed(), "Mobile Responsiveness": self.analyze_mobile_responsiveness(), "Content Quality": self.analyze_content_quality(), "CTA Effectiveness": self.analyze_cta(), "SEO Strategies": self.analyze_seo(), "Social Media Analysis": self.analyze_social_media(), # Added social media analysis "Improvement Suggestions": self.suggest_improvements() } return results # Initialize the LLM with Groq configuration llm = LLM( model="groq/llama-3.2-90b-text-preview", # You can change this to other Groq models api_key=os.environ['GROQ_API_KEY'], base_url="https://api.groq.com/openai/v1", temperature=0 ) llm_mistral = LLM( model="groq/gemma2-9b-it", api_key=os.environ['GROQ_API_KEY'], base_url="https://api.groq.com/openai/v1", temperature=0 ) scrape_tool = ScrapeWebsiteTool() website_analyzer_tool = WebsiteAnalyzerTool() # Define the Competitor Information Analyst agent competitor_analyst = Agent( role="Competitor Information Analyst", goal="Provide in-depth competitor analysis by collecting essential business and website information.", backstory=( "You are an experienced Competitor Information Analyst with 8 years of expertise in identifying market competitors " "and uncovering valuable insights. Your strength lies in analyzing direct, indirect, and replacement competitors, " "highlighting their strengths and weaknesses, pricing strategies, and customer sentiment. You leverage this information " "to provide businesses with actionable strategies for market dominance and improved customer engagement." ), tools=[scrape_tool], llm=llm_mistral, verbose=True, max_iter=5, max_retry_limit=3, ) report_writer = Agent( role="Competitor Report Writer", goal="Generate detailed competitor analysis reports and dashboard insights by synthesizing data from both Web Analyst and Competitor Information Analyst outputs.", backstory="You are responsible for creating insightful and comprehensive markdown reports, summarizing key competitor data including their basic information, market presence, product offerings, pricing strategy, website analysis, and customer sentiment. Your role is to provide actionable insights based on the data collected by the Web Analyst and Competitor Information Analyst.", llm=llm, verbose=True, max_iter=5, # Increase iteration limit for more thorough outputs max_retry_limit=3, # Allow more retries to ensure robust output ) # Define the content analysis task for the Competitor Information Analyst agent content_analysis_task = Task( description=( "Perform a thorough analysis of the website at the provided URL: {url}. Extract key competitor details and relevant business information, " "and provide insights into their market positioning. The analysis should include:" "\n1. **Company Information**: Company name, URL, contact email, phone, and location." "\n2. **Product/Service Offerings**: List of products or services offered, including any unique selling points." "\n3. **Strengths and Weaknesses**: Identify the key strengths and weaknesses of the competitor based on their website content." "\n4. **Target Market**: Analyze the competitor's target audience and market reach." "\n5. **Pricing Strategy**: Provide insights into their pricing structure and strategy (if available)." "\n6. **Customer Testimonials or Sentiment**: Review and summarize customer feedback from testimonials or reviews." "\n7. **User Experience (UX)**: Evaluate the overall user experience of the website, including design and ease of use." "\n8. **Call-to-Action (CTA) Effectiveness**: Assess the clarity and effectiveness of any calls-to-action present on the site." ), expected_output=( "A structured JSON report containing:" "\n- Company Information: Name:\n, URL:\n, contact details:\n, location:\n." "\n- Product/Service Offerings: Detailed list with unique selling points." "\n- Strengths and Weaknesses: Key strengths and weaknesses based on website content." "\n- Target Market: Identified audience and market positioning." "\n- Pricing Strategy: Pricing details and strategy (if available)." "\n- Customer Testimonials or Sentiment: Summary of customer reviews." "\n- UX Assessment: Evaluation of user experience." "\n- CTA Effectiveness: Analysis of call-to-action elements." ), agent=competitor_analyst, ) # Create a task for competitor analysis using the defined model competitor_analysis_task = Task( description=( """Create a detailed markdown report for {url} that includes the following: Competitor's basic information (company name, URL, email, location). Operating markets and regions. Products/services offered and pricing strategy. Strengths and weaknesses in their website and user experience. General customer sentiment from reviews or testimonials. Detailed analysis of their website based on {ux}: UX Assessment Loading Speed Mobile Responsiveness Content Quality CTA Effectiveness SEO Strategies Social Media Analysis Suggestions for improvement. Use the outputs from both Competitor Information Analyst to generate a comprehensive report and JSON summary""" ), agent=report_writer, expected_output=( "A structured markdown report with sections for basic info, operating markets, " "products/services, pricing strategy, website strengths/weaknesses, and customer sentiment. " "Additionally, a JSON summary containing key extracted insights." ), # output_pydantic=CompetitorAnalysis, # Use a custom Pydantic model for structured JSON output context=[content_analysis_task], output_file=f"competitor_analysis.md", ) # Create a crew with the agent and task crew = Crew( agents=[competitor_analyst, report_writer], tasks=[content_analysis_task, competitor_analysis_task], verbose=True, ) # Streamlit application st.set_page_config(page_title="Spark Competitor Analysis", page_icon=":bar_chart:") st.title("BrainSpark Competitor Analysis") # Get the competitor URL from the user competitor_url = st.text_input("Enter the competitor website URL:") if st.button("Analyze Competitor"): if competitor_url: analyzes = WebsiteAnalyzer(url=competitor_url) analyze=analyzes.run_analysis() # Run the competitor analysis task crew_output = crew.kickoff(inputs={'url': competitor_url, 'ux': analyze}) report = crew_output.raw # Display the report st.markdown(report) # Allow the user to download the report st.download_button( label="Download Competitor Analysis Report", data=report, file_name="competitor_analysis.md", mime="text/markdown", ) else: st.error("Please enter a competitor website URL.")