Upload 20 files

ValidationAgent/ValidationAgent.py

@@ -0,0 +1,126 @@
+from agency_swarm.agents import Agent
+from agency_swarm.tools import CodeInterpreter
+import os
+import logging
+from .tools.SearchAndScrape import SearchAndScrape
+
+class ValidationAgent(Agent):
+    def __init__(self):
+        super().__init__(
+            name="ValidationAgent",
+            description="This agent validates market research reports using AI and ensures data completeness.",
+            instructions="./instructions.md",
+            files_folder="./files",
+            schemas_folder="./schemas",
+            tools=[SearchAndScrape],
+            tools_folder="./tools",
+            temperature=0.3,
+            model="groq/llama-3.3-70b-versatile", 
+            max_prompt_tokens=25000,
+        )
+
+    def validate_data(self, report_data):
+        """Validate the report using AI and fill gaps if needed."""
+        validation_prompt = f"""
+        Analyze this market research report for quality and completeness:
+
+        {report_data}
+
+        Please check for:
+        1. Missing key information
+        2. Data accuracy and consistency
+        3. Logical flow and structure
+        4. Completeness of sections:
+           - Market Size & Growth
+           - Competitive Landscape
+           - Consumer Analysis
+           - Technology & Innovation
+           - Future Outlook
+
+        Provide a detailed assessment with:
+        1. Quality score (0-100)
+        2. List of missing or incomplete sections
+        3. Specific recommendations for improvement
+        4. Additional data points needed
+
+        Format: JSON with these keys:
+        {
+            "quality_score": int,
+            "missing_sections": list,
+            "recommendations": list,
+            "additional_data_needed": list,
+            "is_complete": boolean
+        }
+        """
+
+        try:
+            model = self._get_model()  # Updated method to get model
+            response = model.generate_content(validation_prompt)
+            validation_result = response.text
+            
+            # If validation shows missing data, scrape for it
+            if '"is_complete": false' in validation_result.lower():
+                missing_data = self._fill_missing_data(validation_result)
+                if missing_data:
+                    # Combine original report with new data
+                    updated_report = self._merge_reports(report_data, missing_data)
+                    # Validate again
+                    return self.validate_data(updated_report)
+            
+            return validation_result
+
+        except Exception as e:
+            logging.error(f"Validation error: {str(e)}")
+            return {"error": str(e)}
+
+    def _fill_missing_data(self, validation_result):
+        """Fill missing data based on validation results."""
+        try:
+            # Extract missing sections from validation result
+            import json
+            result = json.loads(validation_result)
+            missing_sections = result.get("missing_sections", [])
+            
+            additional_data = []
+            for section in missing_sections:
+                # Create specific search query for missing section
+                search_query = f"{section} market research data analysis"
+                tool = SearchAndScrape(query=search_query)
+                section_data = tool.run()
+                if section_data:
+                    additional_data.append({
+                        "section": section,
+                        "content": section_data
+                    })
+            
+            return additional_data if additional_data else None
+
+        except Exception as e:
+            logging.error(f"Error filling missing data: {str(e)}")
+            return None
+
+    def _merge_reports(self, original_report, new_data):
+        """Merge original report with newly scraped data."""
+        merge_prompt = f"""
+        Merge this original report with new data:
+
+        Original Report:
+        {original_report}
+
+        New Data to Add:
+        {new_data}
+
+        Please create a cohesive, well-structured report that incorporates all information without duplication.
+        Ensure proper flow and transitions between sections.
+        """
+
+        try:
+            model = self._get_model()  # Updated method to get model
+            response = model.generate_content(merge_prompt)
+            return response.text
+        except Exception as e:
+            logging.error(f"Error merging reports: {str(e)}")
+            return original_report
+
+    def response_validator(self, message):
+        return message

ValidationAgent/__init__.py

@@ -0,0 +1 @@
+from .ValidationAgent import ValidationAgent

ValidationAgent/instructions.md

@@ -0,0 +1,12 @@
+# ValidationAgent Instructions
+
+You are an agent responsible for reviewing draft reports for completeness, accuracy, and quality. Your role is to identify gaps and inconsistencies in the data and use Firecrawl to scrape additional data to fill these gaps or improve existing sections.
+
+### Primary Instructions:
+1. Review the draft report provided by the WebScraperAgent for completeness and accuracy.
+2. Identify any gaps or inconsistencies in the data presented in the draft report.
+3. Use Firecrawl to scrape additional data to fill identified gaps or improve existing sections of the report.
+4. Validate the newly gathered data for relevance and accuracy.
+5. Compile the validated data into a final report format.
+6. Communicate with the WebScraperAgent to address any persistent issues or discrepancies in the data.
+7. Ensure the final report is accurate, complete, and ready for submission to the MarketInsightsCEO.

ValidationAgent/tools/DataValidationTool.py

@@ -0,0 +1,54 @@
+from agency_swarm.tools import BaseTool
+from pydantic import Field
+import re
+from collections import defaultdict
+
+class DataValidationTool(BaseTool):
+    """
+    This tool ensures that all data is accurate and compiles it into a cohesive final report.
+    It validates numerical consistency and checks for valid date formats.
+    """
+
+    raw_data: str = Field(
+        ..., description="The raw data to be validated and compiled into the final report."
+    )
+
+    expected_repetitions: dict = Field(
+        default_factory=dict,
+        description="A dictionary specifying numbers that are expected to repeat and their expected counts."
+    )
+
+    def run(self):
+        """
+        Validates the accuracy of the data and compiles it into a cohesive final report.
+        """
+
+        # Validate numerical consistency
+        numbers = re.findall(r'\b\d+\b', self.raw_data)
+        number_counts = defaultdict(int)
+
+        for number in numbers:
+            number_counts[number] += 1
+
+        conflicting_numbers = []
+        for number, count in number_counts.items():
+            expected_count = self.expected_repetitions.get(number, 0)
+            if expected_count == 0:
+                # If no expected count is provided, assume the number should appear once
+                expected_count = 1
+            if count != expected_count:
+                conflicting_numbers.append(f"{number} (found {count}, expected {expected_count})")
+
+        if conflicting_numbers:
+            return f"Data validation failed: Conflicting numerical data found for numbers: {', '.join(conflicting_numbers)}."
+
+        # Check for valid date formats (e.g., YYYY-MM-DD)
+        dates = re.findall(r'\b\d{4}-\d{2}-\d{2}\b', self.raw_data)
+        if not dates:
+            return "Data validation failed: No valid date formats found."
+
+        # Compile the validated data into a final report
+        final_report = f"Final Report:\n\n{self.raw_data}"
+
+        # Return the final report
+        return final_report

ValidationAgent/tools/FirecrawlDataScraperTool.py

@@ -0,0 +1,40 @@
+from agency_swarm.tools import BaseTool
+from pydantic import Field
+import requests
+import os
+
+# Assuming Firecrawl API requires an API key
+firecrawl_api_key = os.getenv("FIRECRAWL_API_KEY")
+firecrawl_base_url = "https://api.firecrawl.com/data"
+
+class FirecrawlDataScraperTool(BaseTool):
+    """
+    This tool interfaces with Firecrawl to gather additional data based on identified gaps or areas needing improvement in the report.
+    """
+
+    gap_description: str = Field(
+        ..., description="Description of the identified gap or area needing improvement in the report."
+    )
+
+    def run(self):
+        """
+        Interfaces with Firecrawl to gather additional data based on the provided gap description.
+        """
+        # Prepare the request to Firecrawl API
+        headers = {
+            "Authorization": f"Bearer {firecrawl_api_key}",
+            "Content-Type": "application/json"
+        }
+        payload = {
+            "query": self.gap_description
+        }
+
+        # Send the request to Firecrawl API
+        response = requests.post(firecrawl_base_url, json=payload, headers=headers)
+
+        # Check if the request was successful
+        if response.status_code == 200:
+            data = response.json()
+            return f"Data gathered from Firecrawl: {data}"
+        else:
+            return f"Failed to gather data from Firecrawl. Status code: {response.status_code}, Error: {response.text}"

ValidationAgent/tools/GapIdentificationTool.py

@@ -0,0 +1,44 @@
+from agency_swarm.tools import BaseTool
+from pydantic import Field
+import re
+
+class GapIdentificationTool(BaseTool):
+    """
+    This tool analyzes the report to find any logical gaps or inconsistencies in the data or narrative.
+    """
+
+    report_content: str = Field(
+        ..., description="The content of the report to be analyzed for logical gaps or inconsistencies."
+    )
+
+    def run(self):
+        """
+        Analyzes the report content to identify logical gaps or inconsistencies in the data or narrative.
+        """
+        # Define patterns or keywords that might indicate logical gaps or inconsistencies
+        gap_indicators = [
+            "however", "but", "although", "nevertheless", "in contrast", "on the other hand"
+        ]
+
+        # Find sentences with potential logical gaps
+        potential_gaps = []
+        sentences = re.split(r'(?<=[.!?]) +', self.report_content)
+        for sentence in sentences:
+            if any(indicator in sentence for indicator in gap_indicators):
+                potential_gaps.append(sentence)
+
+        # Check for inconsistencies in data (e.g., conflicting numbers)
+        # This is a simple example using regex to find numbers
+        numbers = re.findall(r'\b\d+\b', self.report_content)
+        inconsistencies = []
+        if len(set(numbers)) != len(numbers):
+            inconsistencies.append("Conflicting numerical data found.")
+
+        # Compile the analysis results
+        analysis_results = {
+            "potential_gaps": potential_gaps,
+            "inconsistencies": inconsistencies
+        }
+
+        # Return the analysis results as a string
+        return f"Analysis Results: {analysis_results}"

ValidationAgent/tools/ReportReviewTool.py

@@ -0,0 +1,47 @@
+from agency_swarm.tools import BaseTool
+from pydantic import Field
+import re
+
+class ReportReviewTool(BaseTool):
+    """
+    This tool analyzes the content of draft reports, checking for completeness, accuracy, and quality.
+    It identifies any missing sections or errors in the report.
+    """
+
+    report_content: str = Field(
+        ..., description="The content of the draft report to be analyzed."
+    )
+
+    def run(self):
+        """
+        Analyzes the report content for completeness, accuracy, and quality.
+        Identifies missing sections or errors in the report.
+        """
+        # Define the expected sections in a report
+        expected_sections = [
+            "Introduction", "Methodology", "Results", "Discussion", "Conclusion"
+        ]
+
+        # Check for missing sections
+        missing_sections = [
+            section for section in expected_sections if section not in self.report_content
+        ]
+
+        # Check for common errors (e.g., spelling mistakes)
+        # This is a simple example using regex to find repeated words
+        errors = re.findall(r'\b(\w+)\s+\1\b', self.report_content)
+
+        # Check for quality (e.g., length of the report)
+        quality_issues = []
+        if len(self.report_content.split()) < 500:
+            quality_issues.append("The report is too short, consider adding more content.")
+
+        # Compile the analysis results
+        analysis_results = {
+            "missing_sections": missing_sections,
+            "errors": errors,
+            "quality_issues": quality_issues
+        }
+
+        # Return the analysis results as a string
+        return f"Analysis Results: {analysis_results}"

ValidationAgent/tools/SearchAndScrape.py

@@ -0,0 +1,76 @@
+from agency_swarm.tools import BaseTool
+from pydantic import Field
+import logging
+import os
+import time
+
+try:
+    from firecrawl import FirecrawlApp
+except ImportError:
+    raise ImportError(
+        "Required packages not found. Please install them using:\n"
+        "pip install firecrawl"
+    )
+
+# Initialize Firecrawl
+FIRECRAWL_API_KEY = "fc-5fadfeae30314d4ea8a3d9afaa75c493"
+firecrawl_app = FirecrawlApp(api_key=FIRECRAWL_API_KEY)
+
+class SearchAndScrape(BaseTool):
+    """
+    This tool scrapes content using Firecrawl based on a provided query.
+    """
+    
+    query: str = Field(
+        ..., 
+        description="The search query to look for",
+        examples=["market trends in technology 2024", "AI industry statistics"]
+    )
+
+    def scrape_with_retry(self, url, max_retries=3):
+        """Helper function to scrape URL with retry logic"""
+        problematic_domains = [
+            'sparktoro.com', 'j-jdis.com', 'linkedin.com',
+            'facebook.com', 'twitter.com', 'reddit.com', '.pdf'
+        ]
+        
+        if any(domain in url.lower() for domain in problematic_domains):
+            logging.info(f"Skipping problematic URL: {url}")
+            return None
+
+        for attempt in range(max_retries):
+            try:
+                response = firecrawl_app.scrape_url(
+                    url=url,
+                    params={'formats': ['markdown']}
+                )
+                
+                if response and response.get('markdown'):
+                    content = response.get('markdown')
+                    if len(content.strip()) > 200:
+                        return content
+                return None
+            except Exception as e:
+                logging.error(f"Attempt {attempt + 1} failed for {url}: {str(e)}")
+                if attempt < max_retries - 1:
+                    time.sleep(2)
+                continue
+        return None
+
+    def run(self):
+        logging.info(f"Scraping content for query: {self.query}")
+        # Here you would typically have a list of URLs to scrape based on the query.
+        # For this example, we will assume a predefined list of URLs.
+        search_results = ["http://example.com/article1", "http://example.com/article2"]  # Placeholder URLs
+
+        if not search_results:
+            return "No search results found."
+
+        for url in search_results:
+            logging.info(f"Attempting to scrape URL: {url}")
+            content = self.scrape_with_retry(url)
+            if content:
+                logging.info(f"Successfully scraped content from {url}")
+                return f"Content from {url}:\n\n{content}"
+
+        return "Failed to scrape content from any of the search results" 

ValidationAgent/tools/__init__.py

@@ -0,0 +1 @@
+from .SearchAndScrape import SearchAndScrape