Update app.py

app.py

@@ -1,29 +1,14 @@
 # app.py
 import os
-import time
-import json
-import re
 import streamlit as st
-import plotly.graph_objects as go
 from dotenv import load_dotenv
-from crewai import Agent, Crew, Process, Task
-from crewai_tools import SerperDevTool
+from agents import create_research_crew
+from utils import format_json_output, update_progress
+from styles import main_styles
 
 # Load environment variables
 load_dotenv()
 
-# Function to get API keys from HF secrets
-def get_api_keys():
-    """Get API keys from environment or Hugging Face Secrets"""
-    try:
-        serper_key = os.environ.get('SERPER_API_KEY')
-        openai_key = os.environ.get('OPENAI_API_KEY')
-        print("API Keys loaded successfully")
-        return serper_key, openai_key
-    except Exception as e:
-        st.error(f"Error loading API keys: {str(e)}")
-        return None, None
-
 # Set page config
 st.set_page_config(
     page_title="Market Research Generator",
@@ -32,524 +17,40 @@ st.set_page_config(
     initial_sidebar_state="expanded"
 )
 
-# Add all CSS styles
-st.markdown("""
-    <style>
-        /* Progress Bar */
-        .progress-container {
-            width: 100%;
-            background: #f0f0f0;
-            border-radius: 8px;
-            margin: 20px 0;
-        }
-
-        .progress-bar {
-            height: 24px;
-            background: linear-gradient(90deg, #4caf50, #81c784);
-            border-radius: 8px;
-            text-align: center;
-            line-height: 24px;
-            color: white;
-            transition: width 0.5s ease;
-        }
-
-        /* Agent Log Styles */
-        .agent-log {
-            background: #1a1a1a;
-            color: #fff;
-            padding: 20px;
-            border-radius: 10px;
-            font-family: 'Courier New', monospace;
-            height: 500px;
-            overflow-y: auto;
-        }
-        
-        .log-entry {
-            margin: 10px 0;
-            padding: 8px;
-            border-left: 3px solid;
-            animation: fadeIn 0.5s ease-in;
-        }
-        
-        .log-entry.researcher {
-            border-color: #4CAF50;
-            background: rgba(76, 175, 80, 0.1);
-        }
-        
-        .log-entry.analyst {
-            border-color: #2196F3;
-            background: rgba(33, 150, 243, 0.1);
-        }
-        
-        .log-entry.writer {
-            border-color: #FF9800;
-            background: rgba(255, 152, 0, 0.1);
-        }
-        
-        .timestamp {
-            color: #888;
-            font-size: 0.8em;
-        }
-        
-        @keyframes fadeIn {
-            from { opacity: 0; transform: translateY(10px); }
-            to { opacity: 1; transform: translateY(0); }
-        }
-    </style>
-""", unsafe_allow_html=True)
-
-# Initialize session state
-if 'agent_logs' not in st.session_state:
-    st.session_state.agent_logs = []
-if 'agent_outputs' not in st.session_state:
-    st.session_state.agent_outputs = []
-
-def log_agent_activity(agent_type: str, message: str):
-    """Log agent activity with timestamp"""
-    timestamp = time.strftime("%H:%M:%S")
-    st.session_state.agent_logs.append({
-        'timestamp': timestamp,
-        'agent': agent_type,
-        'message': message
-    })
-
-def log_agent_output(agent_type: str, output: str):
-    """Log agent outputs for transparency"""
-    st.session_state.agent_outputs.append({
-        'agent': agent_type,
-        'output': output
-    })
-
-def display_agent_logs():
-    """Display all agent logs"""
-    if not st.session_state.agent_logs:
-        st.info("No agent activity logs yet. Generate a report to see agents in action!")
-        return
-
-    st.markdown('<div class="agent-log">', unsafe_allow_html=True)
-    
-    for log in st.session_state.agent_logs:
-        st.markdown(f"""
-            <div class="log-entry {log['agent'].lower()}">
-                <span class="timestamp">[{log['timestamp']}]</span>
-                <strong>{log['agent']}:</strong> {log['message']}
-            </div>
-        """, unsafe_allow_html=True)
-    
-    st.markdown('</div>', unsafe_allow_html=True)
-
-def display_agent_outputs():
-    """Display all agent outputs"""
-    if not st.session_state.agent_outputs:
-        st.info("No agent outputs available yet. Generate a report to see agents in action!")
-        return
-
-    for output in st.session_state.agent_outputs:
-        st.markdown(f"**{output['agent']} Output:**")
-        st.code(output['output'], language='json')
-
-def update_progress(progress_container, percentage, message=""):
-    """Update the progress bar."""
-    progress_html = f"""
-        <div class="progress-container">
-            <div class="progress-bar" style="width: {percentage}%">{message}</div>
-        </div>
-    """
-    progress_container.markdown(progress_html, unsafe_allow_html=True)
-def create_research_crew(topic: str):
-    """Create the research crew with SerperDev tool integration"""
-    try:
-        # Initialize SerperDev tool
-        search_tool = SerperDevTool()
-        log_agent_activity("System", "Initialized SerperDev search tool")
-
-        researcher = Agent(
-            role='Research Analyst',
-            goal=f'Conduct exhaustive market research about {topic} with comprehensive analysis and multiple data sources',
-            backstory="""You are a senior market research analyst with 15+ years of experience in conducting 
-                detailed industry analysis. You excel at gathering in-depth market data, analyzing industry trends, 
-                and identifying market patterns through multiple verified sources. You always ensure data accuracy 
-                and provide contextual analysis alongside raw data. You're known for producing comprehensive 
-                research reports that combine quantitative data with qualitative insights.""",
-            tools=[search_tool],
-            verbose=True
-        )
-        
-        analyst = Agent(
-            role='Data Analyst',
-            goal='Transform research data into profound market insights with detailed analysis and visualizations',
-            backstory="""You are an expert data analyst and market intelligence specialist with deep expertise 
-                in converting complex market data into actionable insights. You specialize in predictive analytics, 
-                trend analysis, and creating data visualizations that tell compelling stories. You have a 
-                track record of identifying hidden market opportunities and providing nuanced analysis of market dynamics.
-                You excel at breaking down complex market phenomena into understandable insights.""",
-            tools=[search_tool],
-            verbose=True
-        )
-        
-        writer = Agent(
-            role='Report Writer',
-            goal='Create extensive, professional market research reports with compelling narratives and insights',
-            backstory="""You are a seasoned business writer and market research expert who specializes in creating 
-                comprehensive, engaging market research reports. You excel at weaving data and insights into compelling 
-                narratives that provide deep understanding of market dynamics. You're skilled at creating well-structured, 
-                detailed reports that maintain reader engagement while delivering thorough analysis. You're known for your 
-                ability to explain complex market dynamics in clear, professional language while maintaining attention to detail.""",
-            verbose=True
-        )
-
-        research_task = Task(
-            description=f"""
-            Conduct exhaustive market research on {topic} with:
-
-            1. Comprehensive Market Overview:
-                - Detailed market size analysis (historical, current, and projected)
-                - Growth rates and CAGR with supporting rationale
-                - Market segmentation analysis (by type, application, region, etc.)
-                - Value chain analysis
-                - Pricing analysis and trends
-                - Regional market breakdowns with specific data points
-                - Key market metrics and KPIs
-
-            2. In-depth Competitive Analysis:
-                - Detailed profiles of key players
-                - Market share analysis with historical trends
-                - Competitive strategies and positioning
-                - Recent developments and their impact
-                - SWOT analysis of major players
-                - Competitive advantages and differentiators
-                - Market concentration and fragmentation analysis
-
-            3. Extensive Market Dynamics:
-                - Detailed analysis of growth drivers with specific examples
-                - Comprehensive examination of market challenges
-                - Emerging trends with supporting data
-                - Regulatory landscape and its impact
-                - Technology impact assessment
-                - Patent analysis and innovation trends
-                - Consumer behavior analysis
-                - Supply-demand gap analysis
-
-            4. Economic and External Factors:
-                - Macroeconomic influences
-                - Industry-specific economic indicators
-                - Global and regional economic impacts
-                - Trade dynamics and supply chain analysis
-                - Investment patterns and M&A activity
-
-            Use multiple sources for verification and cross-reference all data points.
-            Include case studies and specific examples where relevant.
-            Provide rich context for all numerical data.
-            Format all data to be visualization-ready.
-            """,
-            agent=researcher,
-            expected_output="Exhaustive research data with comprehensive analysis and verified sources"
-        )    
-analysis_task = Task(
-            description="""
-            Perform comprehensive analysis of the research findings:
-
-            1. Strategic Market Analysis:
-                - Detailed 5-year growth projections with scenario analysis
-                - Market maturity assessment
-                - Porter's Five Forces analysis
-                - Market attractiveness analysis
-                - Investment opportunity assessment
-                - Risk analysis and mitigation strategies
-
-            2. Advanced Data Analysis:
-                - Time series analysis of key metrics
-                - Correlation analysis between market factors
-                - Market share evolution analysis
-                - Price sensitivity analysis
-                - Demand forecasting models
-                - Regional market potential assessment
-                - Market penetration analysis
-
-            3. Competitive Intelligence:
-                - Detailed competitor benchmarking
-                - Market positioning matrix
-                - Strategic group mapping
-                - Competitive strategy analysis
-                - Market entry barriers assessment
-                - Future competitive scenarios
-
-            4. Trend Analysis:
-                - Technology adoption curves
-                - Consumer trend analysis
-                - Innovation impact assessment
-                - Market disruption analysis
-                - Future market scenarios
-                - Emerging opportunities assessment
-
-            Create visualization-ready data for:
-                - Market size and growth trends
-                - Competitive landscape maps
-                - Regional distribution charts
-                - Market share evolution
-                - Value chain analysis
-                - Technology adoption curves
-                - Price trend analysis
-                - Market segmentation
-                - Key player comparison matrices
-
-            Include specific metrics, percentages, and growth rates.
-            Provide context and implications for all analyses.
-            Format data for multiple visualization types.
-            """,
-            agent=analyst,
-            expected_output="Comprehensive analysis with visualization-ready metrics and detailed insights",
-            context=[research_task]
-        )
-
-        report_task = Task(
-            description="""
-            Create an extensive professional report including:
+# Add styles
+st.markdown(main_styles, unsafe_allow_html=True)
 
-            1. Executive Summary (3-4 pages):
-                - Key findings and insights
-                - Critical market metrics
-                - Major trends and developments
-                - Strategic implications
-                - Investment opportunities
-                - Future outlook
-                - Key recommendations
-
-            2. Comprehensive Market Analysis (15-20 pages):
-                a) Market Overview
-                    - Market definition and segmentation
-                    - Market size and growth analysis
-                    - Value chain analysis
-                    - Market structure and dynamics
-                    - Regional market analysis
-                    - Pricing analysis
-
-                b) Competitive Landscape
-                    - Market share analysis
-                    - Key player profiles
-                    - Competitive strategies
-                    - Market positioning
-                    - SWOT analysis
-                    - Recent developments
-
-                c) Market Dynamics
-                    - Growth drivers analysis
-                    - Market challenges
-                    - Opportunity assessment
-                    - Trend analysis
-                    - Technology impact
-                    - Regulatory framework
-
-                d) Future Outlook
-                    - Market forecasts
-                    - Trend projections
-                    - Scenario analysis
-                    - Strategic recommendations
-                    - Risk analysis
-                    - Success factors
-
-            3. Appendices
-                - Detailed methodology
-                - Data tables
-                - Additional charts and graphs
-                - Case studies
-                - Glossary
-                - Sources and citations
-
-            Format as JSON:
-            {
-                "exec_summary": {
-                    "summary": "comprehensive executive overview",
-                    "market_size": "detailed market size analysis",
-                    "growth_rate": "detailed growth projections",
-                    "key_players": "detailed competitive analysis",
-                    "critical_insights": "key strategic insights",
-                    "recommendations": "strategic recommendations"
-                },
-                "detailed_report": {
-                    "market_overview": "comprehensive market analysis",
-                    "competitive_landscape": "detailed competitive analysis",
-                    "market_dynamics": "in-depth market dynamics",
-                    "future_outlook": "detailed projections and scenarios",
-                    "strategic_implications": "strategic analysis and recommendations"
-                },
-                "sources": ["source1", "source2"],
-                "metrics": {
-                    "market_size_data": [],
-                    "growth_rates": [],
-                    "market_shares": {},
-                    "regional_distribution": [],
-                    "segment_analysis": [],
-                    "competitor_metrics": {},
-                    "trend_indicators": []
-                },
-                "visualizations": {
-                    "market_growth": "time series data",
-                    "competitive_landscape": "positioning data",
-                    "regional_analysis": "geographical data",
-                    "segment_breakdown": "segmentation data",
-                    "trend_analysis": "trend indicators"
-                }
-            }
-            """,
-            agent=writer,
-            expected_output="Comprehensive JSON containing detailed executive summary and full report",
-            context=[research_task, analysis_task]
-        )
-
-        return Crew(
-            agents=[researcher, analyst, writer],
-            tasks=[research_task, analysis_task, report_task],
-            verbose=True,
-            process=Process.sequential
-        )
-    except Exception as e:
-        st.error(f"Error initializing research crew: {str(e)}")
-        raise e
-def extract_section(text, section_name):
-    """Extract a section from the text"""
-    pattern = f"{section_name}.*?\n(.*?)(?=\n\n|$)"
-    match = re.search(pattern, text, re.DOTALL | re.IGNORECASE)
-    if match:
-        return match.group(1).strip()
-    
-    # Try alternative patterns
-    pattern2 = f"{section_name}[:\s](.*?)(?=\n\n|$)"
-    match = re.search(pattern2, text, re.DOTALL | re.IGNORECASE)
-    return match.group(1).strip() if match else "Information not found"
-
-def extract_sources(text):
-    """Extract sources from the text"""
-    sources = []
-    patterns = [
-        r"Source:.*?(?:\n|$)",
-        r"\[.*?\]",
-        r"\(https?://.*?\)",
-        r"Reference:.*?(?:\n|$)",
-        r"Retrieved from:.*?(?:\n|$)"
-    ]
-    
-    for pattern in patterns:
-        matches = re.finditer(pattern, text, re.MULTILINE)
-        sources.extend([match.group().strip() for match in matches])
-    
-    return sources if sources else ["Sources not explicitly mentioned in the report"]
-
-def format_json_output(raw_output):
-    """Format CrewOutput or raw string into proper JSON structure"""
+def get_api_keys():
+    """Get API keys from environment or Hugging Face Secrets"""
     try:
-        # Handle CrewOutput object
-        if hasattr(raw_output, 'raw_output'):
-            raw_text = str(raw_output.raw_output)
-        else:
-            raw_text = str(raw_output)
-
-        # Try to find and parse JSON structure
-        json_pattern = r"\{[\s\S]*\}"
-        match = re.search(json_pattern, raw_text)
-        if match:
-            try:
-                return json.loads(match.group())
-            except:
-                pass
-
-        # If no JSON found, create structured format
-        return {
-            "exec_summary": {
-                "summary": extract_section(raw_text, "Executive Summary"),
-                "market_size": extract_section(raw_text, "Market Size"),
-                "growth_rate": extract_section(raw_text, "Growth Rate"),
-                "key_players": extract_section(raw_text, "Key Players")
-            },
-            "detailed_report": raw_text,
-            "sources": extract_sources(raw_text),
-            "metrics": {
-                "market_size_data": [],
-                "growth_rates": [],
-                "market_shares": {},
-                "regional_distribution": [],
-                "segment_analysis": [],
-                "competitor_metrics": {},
-                "trend_indicators": []
-            }
-        }
+        serper_key = os.environ.get('SERPER_API_KEY')
+        openai_key = os.environ.get('OPENAI_API_KEY')
+        print("API Keys loaded successfully")
+        return serper_key, openai_key
     except Exception as e:
-        st.error(f"Error formatting output: {str(e)}")
-        return {
-            "exec_summary": {
-                "summary": "Error formatting report",
-                "market_size": "N/A",
-                "growth_rate": "N/A",
-                "key_players": "N/A"
-            },
-            "detailed_report": raw_text if 'raw_text' in locals() else str(raw_output),
-            "sources": [],
-            "metrics": {
-                "market_size_data": [],
-                "growth_rates": [],
-                "market_shares": {}
-            }
-        }
+        st.error(f"Error loading API keys: {str(e)}")
+        return None, None
 
-def run_market_research(topic: str, progress_container, chat_container, log_container):
+def run_market_research(topic: str, progress_container):
     """Run the market research process"""
     try:
-        # Clear previous logs
-        st.session_state.agent_logs = []
-        st.session_state.agent_outputs = []
-        
-        # Initialize research team
-        update_progress(progress_container, 0, "Initializing research team...")
-        log_agent_activity("System", f"Starting market research for: {topic}")
-        
         # Create and run the crew
         crew = create_research_crew(topic)
         
-        # Research Phase
+        # Update progress
         update_progress(progress_container, 25, "Gathering market data...")
-        log_agent_activity("Research Analyst", f"Beginning comprehensive research on {topic}")
-        time.sleep(1)
-        
-        # Analysis Phase
         update_progress(progress_container, 50, "Analyzing findings...")
-        log_agent_activity("Data Analyst", "Processing research data...")
-        time.sleep(1)
-        
-        # Report Phase
         update_progress(progress_container, 75, "Generating report...")
-        log_agent_activity("Report Writer", "Compiling final report...")
-        time.sleep(1)
         
         # Execute the crew
         result = crew.kickoff()
         
-        # Process the result
-        if hasattr(result, 'raw_output'):
-            raw_text = str(result.raw_output)
-        else:
-            raw_text = str(result)
-        
-        # Log the raw output for debugging
-        print("Raw output:", raw_text)
-        log_agent_output("Crew", raw_text)
-        
-        # Format the result
-        report_data = format_json_output(raw_text)
-        
-        # Update progress and log completion
+        # Format and return the result
         update_progress(progress_container, 100, "Report completed!")
-        log_agent_activity("System", "Report generation completed successfully")
-        
-        # Store in session state
-        st.session_state.current_report = report_data
-        st.session_state.current_topic = topic
-        
-        return report_data
+        return format_json_output(result)
 
     except Exception as e:
-        error_msg = f"Error during research: {str(e)}"
-        log_agent_activity("System", f"Error: {error_msg}")
-        st.error(error_msg)
+        st.error(f"Error during research: {str(e)}")
         print(f"Full error details: {str(e)}")  # For debugging
         return None
 
@@ -568,54 +69,39 @@ def main():
         st.session_state.generating = False
     
     # Create tabs
-    tab1, tab2, tab3, tab4 = st.tabs(["Generate Report", "View Reports", "Live Agent Log", "Agent Outputs"])
+    tab1, tab2 = st.tabs(["Generate Report", "View Reports"])
     
     with tab1:
-        col1, col2 = st.columns([2, 3])
-        
-        with col1:
-            st.subheader("Enter Research Topic")
-            topic = st.text_input(
-                "What market would you like to research?",
-                placeholder="e.g., Electric Vehicles Market"
-            )
-            
-            if st.session_state.generating:
-                st.button("Generating Report...", disabled=True)
-            else:
-                if st.button("Generate Report", type="primary"):
-                    if not topic:
-                        st.error("Please enter a research topic")
-                    else:
-                        st.session_state.generating = True
-                        
-                        # Create containers
-                        progress_container = st.container()
-                        chat_container = st.container()
-                        log_container = st.container()
-                        
-                        try:
-                            with st.spinner("Generating report..."):
-                                result = run_market_research(
-                                    topic, 
-                                    progress_container, 
-                                    chat_container, 
-                                    log_container
-                                )
-                                
-                                if result:
-                                    st.success("Report generated successfully! Switch to the Reports tab to view it.")
-                        except Exception as e:
-                            st.error(f"Error generating report: {str(e)}")
-                        finally:
-                            st.session_state.generating = False
+        st.subheader("Enter Research Topic")
+        topic = st.text_input(
+            "What market would you like to research?",
+            placeholder="e.g., Electric Vehicles Market"
+        )
         
-        # Show live agent activity in col2
-        with col2:
-            st.subheader("Live Agent Activity")
-            st.markdown('<div class="agent-log">', unsafe_allow_html=True)
-            display_agent_logs()
-            st.markdown('</div>', unsafe_allow_html=True)
+        if st.session_state.generating:
+            st.button("Generating Report...", disabled=True)
+        else:
+            if st.button("Generate Report", type="primary"):
+                if not topic:
+                    st.error("Please enter a research topic")
+                else:
+                    st.session_state.generating = True
+                    
+                    # Create progress container
+                    progress_container = st.container()
+                    
+                    try:
+                        with st.spinner("Generating comprehensive market research report..."):
+                            result = run_market_research(topic, progress_container)
+                            
+                            if result:
+                                st.session_state.current_report = result
+                                st.session_state.current_topic = topic
+                                st.success("Report generated successfully! View it in the Reports tab.")
+                    except Exception as e:
+                        st.error(f"Error generating report: {str(e)}")
+                    finally:
+                        st.session_state.generating = False
     
     with tab2:
         if 'current_report' in st.session_state:
@@ -623,83 +109,38 @@ def main():
             topic = st.session_state.current_topic
             
             # Display AI Disclaimer
-            st.markdown("""
-                <div class="report-header">
-                    <strong>⚠️ AI-Generated Report Disclaimer</strong>
-                    <p>This report was generated using AI agents for market research and analysis. 
-                    While we strive for accuracy, please review all content and verify critical information 
-                    independently. The analysis and recommendations provided should be used as a 
-                    supplementary resource rather than the sole basis for decision-making.</p>
-                </div>
-            """, unsafe_allow_html=True)
+            st.info("⚠️ This report was generated using AI. Please verify critical information independently.")
             
-            # Executive Summary Section
+            # Executive Summary
             st.subheader("Executive Summary")
-            st.markdown(f"""
-                <div class="exec-summary">
-                    <h4>Summary</h4>
-                    {report['exec_summary']['summary']}
-                    
-                    <div class="key-findings">
-                        <div class="finding-card">
-                            <h4>Market Size</h4>
-                            {report['exec_summary']['market_size']}
-                        </div>
-                        <div class="finding-card">
-                            <h4>Growth Rate</h4>
-                            {report['exec_summary']['growth_rate']}
-                        </div>
-                        <div class="finding-card">
-                            <h4>Key Players</h4>
-                            {report['exec_summary']['key_players']}
-                        </div>
-                    </div>
-                </div>
-            """, unsafe_allow_html=True)
+            st.write(report['exec_summary']['summary'])
+            
+            # Key Metrics
+            col1, col2, col3 = st.columns(3)
+            with col1:
+                st.metric("Market Size", report['exec_summary']['market_size'])
+            with col2:
+                st.metric("Growth Rate", report['exec_summary']['growth_rate'])
+            with col3:
+                st.metric("Key Players", report['exec_summary']['key_players'])
             
-            # Detailed Report Section
-            st.subheader("Detailed Report")
-            st.markdown(f"""
-                <div class="detailed-section">
-                    {report['detailed_report']}
-                </div>
-            """, unsafe_allow_html=True)
+            # Detailed Report
+            st.subheader("Detailed Analysis")
+            st.write(report['detailed_report'])
             
-            # Sources Section
+            # Sources
             if report['sources']:
                 st.subheader("Sources")
                 for source in report['sources']:
                     st.markdown(f"- {source}")
-    
-    with tab3:
-        st.subheader("📋 Agent Activity Log")
-        st.markdown("""
-            This log shows the detailed activities of our AI agents as they work on your market research.
-            Each agent's actions are tracked and timestamped for transparency.
-        """)
-        
-        # Add log controls
-        col1, col2 = st.columns([6, 1])
-        with col1:
-            if st.session_state.agent_logs:
-                st.download_button(
-                    "Download Log",
-                    data="\n".join([f"[{log['timestamp']}] {log['agent']}: {log['message']}" 
-                                  for log in st.session_state.agent_logs]),
-                    file_name="agent_activity_log.txt",
-                    mime="text/plain"
-                )
-        with col2:
-            if st.button("Clear Log"):
-                st.session_state.agent_logs = []
-                st.rerun()
-        
-        # Display the logs
-        display_agent_logs()
-    
-    with tab4:
-        st.subheader("Agent Outputs")
-        display_agent_outputs()
+            
+            # Download button
+            st.download_button(
+                "Download Report",
+                data=str(report['detailed_report']),
+                file_name=f"{topic.lower().replace(' ', '_')}_market_research.md",
+                mime="text/markdown"
+            )
 
 if __name__ == "__main__":
     main()