"""
Competitor Analysis Routes
Provides AI-powered competitor analysis using web scraping and LLM integration.
"""
import logging
from datetime import datetime, timedelta
from typing import List, Optional
from uuid import uuid4
from fastapi import Depends, HTTPException, Request
from pydantic import BaseModel, ConfigDict, Field
from sqlalchemy.orm import Session
from core.base_routes import BaseAPIRouter
from core.database import get_db
from core.llm_service import LLMService
from core.models import User, CompetitorAnalysis, OAuthToken
from core.security_dependencies import get_current_user
from integrations.notion_service import NotionService
router = BaseAPIRouter(prefix="/api/v1/analysis", tags=["competitor-analysis"])
logger = logging.getLogger(__name__)
# Request/Response Models
class CompetitorAnalysisRequest(BaseModel):
"""Competitor analysis request"""
competitors: List[str] = Field(..., min_length=1, max_length=10, description="List of competitor names/URLs")
analysis_depth: str = Field("standard", description="Analysis depth: basic, standard, comprehensive")
focus_areas: Optional[List[str]] = Field(
default=["products", "pricing", "marketing", "strengths", "weaknesses"],
description="Areas to focus analysis on"
)
notion_database_id: Optional[str] = Field(None, description="Notion database ID for results")
model_config = ConfigDict(extra="allow")
class CompetitorInsight(BaseModel):
"""Individual competitor insight"""
competitor: str
strengths: List[str]
weaknesses: List[str]
market_position: str
key_products: List[str]
pricing_strategy: str
marketing_tactics: List[str]
recent_news: List[str]
class CompetitorAnalysisResponse(BaseModel):
"""Competitor analysis response"""
analysis_id: str
status: str
insights: dict[str, CompetitorInsight]
comparison_matrix: dict
recommendations: List[str]
created_at: datetime
async def fetch_competitor_data(competitor: str, focus_areas: List[str]) -> dict:
"""
Fetch data about a competitor using web scraping and APIs.
In production, this would:
- Scrape the competitor's website
- Query business databases (Crunchbase, LinkedIn)
- Analyze social media presence
- Check recent news and press releases
"""
try:
import httpx
# Simulated competitor data for development
# In production, replace with actual scraping/API calls
competitor_lower = competitor.lower()
# Basic web scraping (if competitor is a URL)
if competitor.startswith("http"):
try:
async with httpx.AsyncClient() as client:
response = await client.get(competitor, timeout=10.0)
if response.status_code == 200:
# Extract basic info from HTML
html = response.text
# Simple title extraction
title_start = html.find("
") + 7
title_end = html.find("", title_start)
title = html[title_start:title_end] if title_start > 6 and title_end > title_start else competitor
return {
"name": title.strip(),
"url": competitor,
"data_source": "web_scrape",
}
except Exception as e:
logger.warning(f"Failed to scrape {competitor}: {e}")
# Return simulated data for development
return {
"name": competitor,
"url": f"https://www.{competitor_lower.replace(' ', '')}.com",
"data_source": "simulated",
"note": "Replace with actual scraping in production"
}
except Exception as e:
logger.error(f"Error fetching competitor data for {competitor}: {e}")
return {
"name": competitor,
"url": None,
"data_source": "error",
"error": str(e)
}
async def analyze_with_llm(competitor_data: dict, focus_areas: List[str], db: Session) -> CompetitorInsight:
"""
Analyze competitor data using LLM to generate insights.
Uses LLMService for cost-optimized provider selection with usage tracking.
Falls back to simulated insights if LLM fails.
"""
competitor_name = competitor_data.get("name", "Unknown")
# Prepare comprehensive prompt
prompt = f"""
Analyze the competitor "{competitor_name}" and provide strategic insights.
Focus Areas: {', '.join(focus_areas)}
Available Data: {competitor_data}
Provide specific, actionable insights including:
- Key competitive advantages (strengths)
- Vulnerabilities and areas for improvement (weaknesses)
- Current market position and strategy
- Main products or services
- Pricing approach and strategy
- Marketing and sales tactics
- Recent notable developments or news
Be specific and data-driven. Avoid generic statements.
"""
system_instruction = """You are an expert business analyst and competitive intelligence specialist.
You provide detailed, specific, and actionable competitor insights.
Your analysis is data-driven, strategic, and focused on business implications."""
try:
# Use LLMService for structured output with usage tracking
llm = LLMService(workspace_id="default", db=db)
result = await llm.generate_structured(
prompt=prompt,
system_instruction=system_instruction,
response_model=CompetitorInsight,
temperature=0.3, # Lower temp for consistency
task_type="analysis", # Enables complexity-based routing
agent_id=None # No agent tracking for this endpoint
)
if result:
logger.info(f"Generated LLM insights for competitor: {competitor_name}")
return result
else:
logger.warning(f"LLM returned None for {competitor_name}, using fallback")
except Exception as e:
logger.error(f"LLM analysis failed for {competitor_name}: {e}")
# Fallback to simulated insights if LLM fails
logger.info(f"Using fallback insights for competitor: {competitor_name}")
return _generate_fallback_insights(competitor_name, focus_areas)
def _generate_fallback_insights(competitor_name: str, focus_areas: List[str]) -> CompetitorInsight:
"""Generate fallback insights when LLM is unavailable."""
return CompetitorInsight(
competitor=competitor_name,
strengths=[
f"Established market presence",
f"Brand recognition in industry",
f"Diverse product offerings",
],
weaknesses=[
f"Limited recent innovation visible",
f"Pricing may not be competitive",
f"Slower technology adoption",
],
market_position=f"Established player competing in key segments",
key_products=[
f"Core product suite",
f"Enterprise solutions",
f"Cloud-based services",
],
pricing_strategy="Market-aligned pricing with enterprise discounts",
marketing_tactics=[
"Digital marketing campaigns",
"Industry partnerships",
"Content marketing strategy",
],
recent_news=[
f"{competitor_name} continues market operations",
f"Product line expansions ongoing",
f"Strategic partnerships maintained",
]
)
def generate_comparison_matrix(insights: dict[str, CompetitorInsight]) -> dict:
"""Generate a comparison matrix across all competitors."""
competitors = list(insights.keys())
comparison = {
"pricing": {},
"market_position": {},
"innovation": {},
"strengths_count": {},
"weaknesses_count": {},
}
for comp in competitors:
insight = insights[comp]
# Count strengths and weaknesses
comparison["strengths_count"][comp] = len(insight.strengths)
comparison["weaknesses_count"][comp] = len(insight.weaknesses)
# Categorize pricing
pricing = insight.pricing_strategy.lower()
if "premium" in pricing:
comparison["pricing"][comp] = "Premium"
elif "budget" in pricing or "low" in pricing:
comparison["pricing"][comp] = "Budget"
else:
comparison["pricing"][comp] = "Mid-range"
# Categorize market position
market = insight.market_position.lower()
if "leader" in market or "dominant" in market:
comparison["market_position"][comp] = "Leader"
elif "challenger" in market or "growing" in market:
comparison["market_position"][comp] = "Challenger"
else:
comparison["market_position"][comp] = "Follower"
# Innovation score (based on recent news)
comparison["innovation"][comp] = "Moderate" if len(insight.recent_news) > 2 else "Low"
return comparison
def generate_recommendations(insights: dict[str, CompetitorInsight], comparison: dict) -> List[str]:
"""Generate strategic recommendations based on analysis."""
recommendations = []
# Analyze pricing gaps
pricing_values = list(comparison["pricing"].values())
if "Premium" in pricing_values and "Budget" in pricing_values:
recommendations.append(
"Consider mid-tier pricing strategy to capture customers between premium and budget competitors"
)
# Analyze market positioning
market_positions = list(comparison["market_position"].values())
if market_positions.count("Follower") >= len(market_positions) / 2:
recommendations.append(
"Market has many followers - consider differentiation strategy to become a challenger"
)
# Analyze strengths commonalities
all_strengths = []
for insight in insights.values():
all_strengths.extend(insight.strengths)
if "brand recognition" in " ".join(all_strengths).lower():
recommendations.append(
"Invest in brand building to compete with established players' strong brand recognition"
)
# Innovation recommendations
innovation_scores = list(comparison["innovation"].values())
if innovation_scores.count("Low") >= len(innovation_scores) / 2:
recommendations.append(
"Opportunity to differentiate through innovation - many competitors show low innovation activity"
)
# Default recommendation if none generated
if not recommendations:
recommendations.append(
"Focus on unique value proposition and customer experience to differentiate from competitors"
)
return recommendations
async def export_competitor_analysis_to_notion(
analysis: CompetitorAnalysis,
notion_token: str
) -> Optional[str]:
"""
Export competitor analysis to Notion database.
Creates a page in the Notion database with the competitor analysis summary.
Args:
analysis: CompetitorAnalysis database model
notion_token: Notion API access token
Returns:
Notion page ID if successful, None otherwise
"""
try:
notion = NotionService(access_token=notion_token)
# Create parent reference to database
parent = {"type": "database_id", "database_id": analysis.notion_database_id}
# Create properties for the page
competitors_str = ", ".join(analysis.competitors)
properties = {
"Competitors": {
"title": [
{
"text": {
"content": f"Competitor Analysis: {competitors_str}"
}
}
]
},
"Analysis Depth": {
"select": {
"name": analysis.analysis_depth.capitalize()
}
},
"Status": {
"select": {
"name": analysis.status.capitalize()
}
},
"Created": {
"date": {
"start": analysis.created_at.isoformat()
}
}
}
# Create children blocks
children = []
# Add comparison matrix section
if analysis.comparison_matrix:
children.append({
"object": "block",
"type": "heading_2",
"heading_2": {
"rich_text": [{"type": "text", "text": {"content": "📊 Comparison Matrix"}}]
}
})
for category, values in analysis.comparison_matrix.items():
children.append({
"object": "block",
"type": "heading_3",
"heading_3": {
"rich_text": [{"type": "text", "text": {"content": category.capitalize()}}]
}
})
for comp, value in values.items():
children.append({
"object": "block",
"type": "bulleted_list_item",
"bulleted_list_item": {
"rich_text": [
{"type": "text", "text": {"content": f"{comp}: "}},
{"type": "text", "text": {"content": str(value)}, "bold": True}
]
}
})
# Add recommendations section
if analysis.recommendations:
children.append({
"object": "block",
"type": "heading_2",
"heading_2": {
"rich_text": [{"type": "text", "text": {"content": "💡 Recommendations"}}]
}
})
for i, rec in enumerate(analysis.recommendations, 1):
children.append({
"object": "block",
"type": "numbered_list_item",
"numbered_list_item": {
"rich_text": [{"type": "text", "text": {"content": rec}}]
}
})
# Create the page
result = notion.create_page(parent, properties, children)
if result and "id" in result:
logger.info(f"Competitor analysis exported to Notion: page_id={result['id']}")
return result["id"]
else:
logger.warning("Notion page creation returned no ID")
return None
except Exception as e:
logger.error(f"Failed to export competitor analysis to Notion: {e}")
return None
@router.post("/competitors", response_model=CompetitorAnalysisResponse)
async def analyze_competitors(
request: Request,
payload: CompetitorAnalysisRequest,
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db)
):
"""
Analyze competitors using AI and web scraping.
Fetches data about each competitor, analyzes using LLM,
and generates actionable insights and recommendations.
Focus Areas:
- products: Product offerings and features
- pricing: Pricing strategies and positioning
- marketing: Marketing channels and tactics
- strengths: Competitive advantages
- weaknesses: Areas for improvement
Uses BYOK handler for cost-optimized LLM integration with automatic fallback.
Results are cached for 7 days to avoid repeated analysis.
"""
try:
# Validate competitors list
if not payload.competitors or len(payload.competitors) == 0:
raise HTTPException(
status_code=400,
detail="At least one competitor must be specified"
)
if len(payload.competitors) > 10:
raise HTTPException(
status_code=400,
detail="Maximum 10 competitors allowed per analysis"
)
# Validate analysis depth
valid_depths = ["basic", "standard", "comprehensive"]
if payload.analysis_depth not in valid_depths:
raise HTTPException(
status_code=400,
detail=f"Invalid analysis depth. Must be one of: {', '.join(valid_depths)}"
)
# Check for recent cached analysis (within 7 days)
cache_expiry = datetime.utcnow() - timedelta(days=7)
cached_analysis = db.query(CompetitorAnalysis).filter(
CompetitorAnalysis.user_id == current_user.id,
CompetitorAnalysis.competitors == payload.competitors, # JSON comparison
CompetitorAnalysis.analysis_depth == payload.analysis_depth,
CompetitorAnalysis.created_at >= cache_expiry
).first()
if cached_analysis:
logger.info(f"Returning cached analysis: {cached_analysis.id}")
# Convert insights dict back to CompetitorInsight objects
insights = {
k: CompetitorInsight(**v) if isinstance(v, dict) else v
for k, v in cached_analysis.insights.items()
}
return CompetitorAnalysisResponse(
analysis_id=cached_analysis.id,
status="cached",
insights=insights,
comparison_matrix=cached_analysis.comparison_matrix,
recommendations=cached_analysis.recommendations,
created_at=cached_analysis.created_at
)
# Generate analysis ID
analysis_id = str(uuid4())
logger.info(
f"Starting competitor analysis: user={current_user.id}, "
f"analysis_id={analysis_id}, "
f"competitors={len(payload.competitors)}"
)
# Fetch data for each competitor
insights = {}
for competitor in payload.competitors:
try:
# Fetch competitor data
competitor_data = await fetch_competitor_data(competitor, payload.focus_areas)
# Analyze with LLM
insight = await analyze_with_llm(competitor_data, payload.focus_areas, db)
insights[competitor] = insight
except Exception as e:
logger.error(f"Failed to analyze competitor {competitor}: {e}")
# Create fallback insight
insights[competitor] = CompetitorInsight(
competitor=competitor,
strengths=[],
weaknesses=[f"Analysis failed: {str(e)}"],
market_position="Unknown",
key_products=[],
pricing_strategy="Unknown",
marketing_tactics=[],
recent_news=[]
)
# Generate comparison matrix
comparison_matrix = generate_comparison_matrix(insights)
# Generate recommendations
recommendations = generate_recommendations(insights, comparison_matrix)
# Convert insights to dict for JSON storage
insights_dict = {k: v.model_dump() if hasattr(v, 'model_dump') else v.__dict__ for k, v in insights.items()}
# Save to database
competitor_analysis = CompetitorAnalysis(
id=analysis_id,
user_id=current_user.id,
competitors=payload.competitors,
analysis_depth=payload.analysis_depth,
focus_areas=payload.focus_areas,
insights=insights_dict,
comparison_matrix=comparison_matrix,
recommendations=recommendations,
notion_database_id=payload.notion_database_id,
notion_page_id=None,
status="complete",
cache_expiry=datetime.utcnow() + timedelta(days=7)
)
db.add(competitor_analysis)
db.commit()
# Log successful analysis
logger.info(
f"Competitor analysis complete: analysis_id={analysis_id}, "
f"competitors_analyzed={len(insights)}, "
f"recommendations={len(recommendations)}"
)
# Export to Notion if notion_database_id provided
if payload.notion_database_id:
logger.info(
f"Notion export requested: database_id={payload.notion_database_id}"
)
# Get Notion OAuth token for the user
notion_token_record = db.query(OAuthToken).filter(
OAuthToken.user_id == current_user.id,
OAuthToken.provider == "notion",
OAuthToken.status == "active"
).first()
if notion_token_record and notion_token_record.access_token:
notion_page_id = await export_competitor_analysis_to_notion(
analysis=competitor_analysis,
notion_token=notion_token_record.access_token
)
if notion_page_id:
# Update the analysis with the Notion page ID
competitor_analysis.notion_page_id = notion_page_id
db.commit()
logger.info(f"Competitor analysis exported to Notion: page_id={notion_page_id}")
else:
logger.warning("Notion export failed, but analysis was saved successfully")
else:
logger.warning(f"No active Notion token found for user {current_user.id}, skipping export")
return CompetitorAnalysisResponse(
analysis_id=analysis_id,
status="complete",
insights=insights,
comparison_matrix=comparison_matrix,
recommendations=recommendations,
created_at=competitor_analysis.created_at
)
except HTTPException:
raise
except Exception as e:
logger.error(f"Competitor analysis failed: {e}", exc_info=True)
raise HTTPException(
status_code=500,
detail=f"Failed to analyze competitors: {str(e)}"
)
@router.get("/competitors/{analysis_id}")
async def get_analysis_result(
analysis_id: str,
request: Request,
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db)
):
"""
Retrieve a previously generated competitor analysis.
"""
# Query database for analysis
analysis = db.query(CompetitorAnalysis).filter(
CompetitorAnalysis.id == analysis_id
).first()
if not analysis:
raise HTTPException(
status_code=404,
detail=f"Competitor analysis with ID '{analysis_id}' not found"
)
# Verify ownership
if analysis.user_id != current_user.id:
raise HTTPException(
status_code=403,
detail="You do not have permission to access this analysis"
)
# Check if cache has expired
if analysis.cache_expiry and analysis.cache_expiry < datetime.utcnow():
analysis.status = "expired"
db.commit()
# Convert insights dict back to CompetitorInsight objects
insights = {
k: CompetitorInsight(**v) if isinstance(v, dict) else v
for k, v in analysis.insights.items()
}
return CompetitorAnalysisResponse(
analysis_id=analysis.id,
status=analysis.status,
insights=insights,
comparison_matrix=analysis.comparison_matrix,
recommendations=analysis.recommendations,
created_at=analysis.created_at
)
@router.get("/competitors")
async def list_analyses(
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
limit: int = 20,
offset: int = 0
):
"""
List all competitor analyses for the current user.
"""
# Query analyses for current user
analyses = db.query(CompetitorAnalysis).filter(
CompetitorAnalysis.user_id == current_user.id
).order_by(
CompetitorAnalysis.created_at.desc()
).offset(offset).limit(limit).all()
total = db.query(CompetitorAnalysis).filter(
CompetitorAnalysis.user_id == current_user.id
).count()
return {
"analyses": [
{
"analysis_id": analysis.id,
"competitors": analysis.competitors,
"analysis_depth": analysis.analysis_depth,
"status": analysis.status,
"created_at": analysis.created_at,
"cache_expiry": analysis.cache_expiry
}
for analysis in analyses
],
"total": total,
"limit": limit,
"offset": offset
}
@router.delete("/competitors/{analysis_id}")
async def delete_analysis(
analysis_id: str,
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db)
):
"""
Delete a competitor analysis.
"""
# Query analysis
analysis = db.query(CompetitorAnalysis).filter(
CompetitorAnalysis.id == analysis_id
).first()
if not analysis:
raise HTTPException(
status_code=404,
detail=f"Competitor analysis with ID '{analysis_id}' not found"
)
# Verify ownership
if analysis.user_id != current_user.id:
raise HTTPException(
status_code=403,
detail="You do not have permission to delete this analysis"
)
# Delete analysis
db.delete(analysis)
db.commit()
logger.info(f"Competitor analysis deleted: analysis_id={analysis_id}")
return {
"success": True,
"message": "Competitor analysis deleted successfully"
}
@router.get("/competitors/templates")
async def list_analysis_templates():
"""
List available competitor analysis templates.
Pre-configured focus areas for different industries/use cases.
"""
templates = {
"ecommerce": {
"name": "E-commerce",
"focus_areas": ["products", "pricing", "shipping", "user_experience", "reviews"],
"description": "Analyze e-commerce competitors"
},
"saas": {
"name": "SaaS",
"focus_areas": ["features", "pricing", "integration", "support", "security"],
"description": "Analyze software-as-a-service competitors"
},
"retail": {
"name": "Retail",
"focus_areas": ["products", "pricing", "locations", "inventory", "loyalty"],
"description": "Analyze retail competitors"
},
"agency": {
"name": "Agency/Services",
"focus_areas": ["services", "pricing", "portfolio", "reputation", "case_studies"],
"description": "Analyze service-based business competitors"
}
}
return {
"templates": templates,
"total": len(templates)
}