Update app.py

app.py

@@ -28,13 +28,40 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
 from urllib.parse import urlparse, urljoin
 import time
 import random
+from functools import wraps
+from pathlib import Path
+from dotenv import load_dotenv
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+from transformers import pipeline
+import torch
+
+load_dotenv()  # Load environment variables from .env file
 
 app = Flask(__name__)
-CORS(app)
+CORS(app, resources={
+    r"/*": {
+        "origins": "*",
+        "methods": ["GET", "POST", "OPTIONS"],
+        "allow_headers": ["Content-Type"]
+    }
+})
 
 # Set up logging
 logging.basicConfig(level=logging.INFO)
 
+# Get the directory containing app.py
+BASE_DIR = Path(__file__).resolve().parent
+
+# Load environment variables from .env file
+env_path = BASE_DIR / '.env'
+load_dotenv(env_path)
+
+# Add debug logging to check environment variables
+print(f"Current working directory: {os.getcwd()}")
+print(f"Env file exists: {os.path.exists('.env')}")
+print(f"GOOGLE_API_KEY value: {os.getenv('GOOGLE_API_KEY')}")
+
 # Set your Google API key securely using an environment variable
 google_api_key = os.getenv('GOOGLE_API_KEY')
 genai.configure(api_key=google_api_key)
@@ -99,6 +126,9 @@ ENERGY_COMPANIES = [
     "https://www.orano.group/en/"
 ]
 
+# Initialize local summarization pipeline (using facebook/bart-large-cnn)
+local_summarizer = pipeline("summarization", model="facebook/bart-large-cnn")
+
 def allowed_file(filename):
     return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
 
@@ -207,237 +237,345 @@ def analyze_document(text):
     return response.text
 
 def process_query(query, role=None, file_id=None):
-    logging.info(f"Processing query: {query}, role: {role}, file_id: {file_id}")
     if file_id:
         return answer_query_from_document(query, file_id)
     else:
         system_prompt = f"You are an AI assistant specializing in {role}." if role else "You are a helpful AI assistant."
+        prompt = f"""
+{system_prompt}
+
+Please format your response using markdown with proper structure:
+- Use '##' for main headings
+- Use '**' for bold text
+- Use bullet points ('*') for lists
+- Add proper spacing between sections
+- Structure the content hierarchically
+- Use proper paragraphs with line breaks
+
+Query: "{query}"
+
+Remember to:
+- Format the response clearly and professionally
+- Use headings for different sections
+- Break down complex information into digestible parts
+- Use bold text for emphasis on key terms
+- Maintain consistent spacing
+"""
+        try:
+            response = model.generate_content(prompt)
+            return response.text
+        except Exception as e:
+            logging.error(f"Error generating content: {str(e)}", exc_info=True)
+            raise e
 
-        prompt = f'''
-        {system_prompt}
-        Query: "{query}"
-        Requirements:
-        - Use a friendly yet professional tone.
-        - Ensure the response is accurate and directly addresses the question.
-        - Include relevant examples, definitions, or comparisons to enhance clarity.
-        - Format the response in well-structured paragraphs or bullet points with bold headings when appropriate.
-        - Use markdown formatting for code snippets, emphasis, and structure.
-        - Aim for a comprehensive response that fully addresses the query.
-        '''
-
-        logging.info("Generating content...")
-        response = model.generate_content(prompt)
-        generated_text = response.text
-        logging.info("Content generated successfully.")
-
-        return generated_text
+def local_summarize(text):
+    """
+    Summarizes the given text using a local huggingface model.
+    If the text is too long, it splits the text into manageable chunks.
+    """
+    # Maximum number of words (roughly) that the summarizer can handle
+    max_words = 800  
+    words = text.split()
+    
+    if len(words) > max_words:
+        # Split text into chunks of ~max_words tokens
+        chunks = []
+        chunk = []
+        for word in words:
+            chunk.append(word)
+            if len(chunk) >= max_words:
+                chunks.append(" ".join(chunk))
+                chunk = []
+        if chunk:
+            chunks.append(" ".join(chunk))
+        
+        summaries = []
+        for chunk in chunks:
+            summary = local_summarizer(chunk, max_length=150, min_length=40, do_sample=False)[0]['summary_text']
+            summaries.append(summary)
+        return " ".join(summaries)
+    else:
+        return local_summarizer(text, max_length=150, min_length=40, do_sample=False)[0]['summary_text']
 
 def scrape_company_news(url):
+    """
+    Scrapes the top company news items from the given URL.
+    Uses a session with retries to mitigate timeouts or transient errors.
+    """
     try:
-        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}
-        response = requests.get(url, headers=headers, timeout=10)
+        session = requests.Session()
+        retries = Retry(total=3, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504])
+        adapter = HTTPAdapter(max_retries=retries)
+        session.mount('https://', adapter)
+        session.mount('http://', adapter)
+        
+        headers = {
+            'User-Agent': ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
+                           'AppleWebKit/537.36 (KHTML, like Gecko) '
+                           'Chrome/91.0.4472.124 Safari/537.36')
+        }
+        response = session.get(url, headers=headers, timeout=10)
         soup = BeautifulSoup(response.content, 'html.parser')
         
-        # This is a basic scraper. You'll need to adjust the selectors for each website
-        articles = soup.find_all('article') or soup.find_all('div', class_='news-item') or soup.find_all('div', class_='press-release')
+        # Combine results from multiple selectors
+        articles = soup.find_all('article')
+        articles.extend(soup.find_all('div', class_='news-item'))
+        articles.extend(soup.find_all('div', class_='press-release'))
         
         news_items = []
-        for article in articles[:5]:  # Limit to top 5 news items per company
-            title = article.find('h2') or article.find('h3') or article.find('a')
-            link = article.find('a')
-            if title and link:
+        # Only take the first 5 items (adjust as needed)
+        for article in articles[:5]:
+            title_elem = article.find('h2') or article.find('h3') or article.find('a')
+            link_elem = article.find('a')
+            if title_elem and link_elem and link_elem.has_attr('href'):
                 news_items.append({
-                    'title': title.text.strip(),
-                    'url': urljoin(url, link['href']),
+                    'title': title_elem.get_text(strip=True),
+                    'url': urljoin(url, link_elem['href']),
                     'source': urlparse(url).netloc
                 })
-        
         return news_items
     except Exception as e:
         logging.error(f"Error scraping {url}: {str(e)}")
         return []
 
-def get_company_news():
-    with ThreadPoolExecutor(max_workers=10) as executor:
-        future_to_url = {executor.submit(scrape_company_news, url): url for url in ENERGY_COMPANIES}
-        all_company_news = []
-        for future in as_completed(future_to_url):
-            all_company_news.extend(future.result())
-            time.sleep(random.uniform(0.5, 1.5))  # Random delay to avoid overwhelming servers
-    return all_company_news
-
 def get_energy_news(query):
+    """
+    Fetches the latest news articles from NewsData.io API based on the query.
+    """
+    logging.info(f"Starting news fetch for query: {query}")
+    
+    news_data_api_key = os.getenv('NEWSDATA_API_KEY')
+    if not news_data_api_key:
+        logging.error("NewsData API key not found in environment variables")
+        return []
+        
+    endpoint = "https://newsdata.io/api/1/news"
+    
+    params = {
+        'apikey': news_data_api_key,
+        'q': query,
+        'country': 'us',
+        'language': 'en',
+        'category': 'business'
+    }
+    
+    logging.info(f"Making API request to: {endpoint}")
+    logging.info(f"With parameters: {params}")
+    
     try:
-        articles = newsapi.get_everything(q=query, language='en', sort_by='publishedAt', page_size=20)
-        return articles['articles']
+        response = requests.get(endpoint, params=params, timeout=10)
+        logging.info(f"API Response status code: {response.status_code}")
+        
+        response.raise_for_status()
+        data = response.json()
+        
+        if data.get("status") == "success":
+            articles = data.get("results", [])
+            logging.info(f"Successfully fetched {len(articles)} articles")
+            return articles
+        else:
+            logging.error(f"NewsData API error response: {data}")
+            return []
+            
     except Exception as e:
         logging.error(f"Error fetching news: {e}")
         return []
 
-def analyze_news_item(item, query, is_company_news=False):
-    source = item.get('source', {}).get('name') if not is_company_news else item.get('source')
-    title = item.get('title', 'No title')
-    content = item.get('description', '') or item.get('content', '') or ''
-    url = item.get('url', '#')
-
-    prompt = f"""
-    Analyze the following news item in the context of the energy market:
-    Query: {query}
-    Source: {source}
-    Title: {title}
-    Content: {content}
-    URL: {url}
-
-    Is this news item directly relevant to "{query}" in the context of the energy market? 
-    Answer ONLY 'YES' or 'NO', followed by a brief explanation.
-    If YES, provide:
-    1. A concise 2-3 sentence summary of the news.
-    2. Key points (up to 3 bullet points).
-    3. Specific impact on the energy market related to {query} (1-2 sentences).
+def robust_analyze_news_item(item, query):
     """
-
+    Analyzes a news item using the generative model with better error handling.
+    """
+    logging.info(f"Starting analysis for article: {item.get('title', 'No title')}")
+    
     try:
-        response = model.generate_content(prompt)
-        analysis = response.text.strip()
-
-        if analysis.startswith("YES"):
-            return {
-                'title': title,
-                'link': url,
-                'source': source,
-                'analysis': analysis.split("YES", 1)[1].strip(),
-                'is_company_news': is_company_news
-            }
-        return None
+        # Extract article information
+        title = item.get('title', '')
+        content = item.get('description', '') or item.get('content', '')
+        source = item.get('source_id', 'Unknown Source')
+        url = item.get('link', '#')
+        
+        logging.info(f"Article details - Title: {title[:100]}...")
+        logging.info(f"Content length: {len(content)} characters")
+        
+        # Skip if no meaningful content
+        if not content or len(content.strip()) < 10:
+            logging.warning(f"Skipping article due to insufficient content: {title}")
+            return None
+
+        prompt = f"""
+        Analyze this news article:
+        Title: {title}
+        Content: {content}
+        
+        Provide a brief analysis in the following format:
+        1. Summary (2-3 sentences)
+        2. Key Points (up to 3 bullet points)
+        3. Market Impact (1-2 sentences about potential market implications)
+        """
+
+        logging.info("Attempting analysis with generative model")
+        try:
+            response = model.generate_content(prompt)
+            analysis = response.text.strip()
+            logging.info("Successfully generated analysis with model")
+        except Exception as e:
+            logging.warning(f"Generative model failed: {str(e)}")
+            logging.info("Falling back to local summarizer")
+            analysis = local_summarize(content)
+
+        result = {
+            'title': title,
+            'link': url,
+            'source': source,
+            'analysis': analysis
+        }
+        logging.info("Successfully created analysis result")
+        return result
+        
     except Exception as e:
-        logging.error(f"Error analyzing news item: {e}")
+        logging.error(f"Error in robust_analyze_news_item: {str(e)}")
         return None
 
+def get_company_news():
+    with ThreadPoolExecutor(max_workers=10) as executor:
+        future_to_url = {executor.submit(scrape_company_news, url): url for url in ENERGY_COMPANIES}
+        all_company_news = []
+        for future in as_completed(future_to_url):
+            all_company_news.extend(future.result())
+            time.sleep(random.uniform(0.5, 1.5))  # Random delay to avoid overwhelming servers
+    return all_company_news
+
 def filter_and_analyze_news(query, articles, company_news):
+    """
+    Processes both News API results and scraped company news.
+    Uses robust_analyze_news_item so that any API errors are handled gracefully.
+    """
     all_news = articles + company_news
     filtered_and_analyzed_news = []
 
     with ThreadPoolExecutor(max_workers=20) as executor:
-        future_to_item = {executor.submit(analyze_news_item, item, query, isinstance(item, dict)): item for item in all_news}
-        for future in as_completed(future_to_item):  # Changed from future_to_url to future_to_item
+        future_to_item = {
+            executor.submit(robust_analyze_news_item, item, query): item
+            for item in all_news
+        }
+        for future in as_completed(future_to_item):
             result = future.result()
             if result:
                 filtered_and_analyzed_news.append(result)
-            
             if len(filtered_and_analyzed_news) >= 20:
                 break
 
     return filtered_and_analyzed_news
 
 def generate_market_summary(query, filtered_news):
+    """
+    Generates an overall market summary using the individual news analyses.
+    Uses the generative model but falls back to local summarization in case of errors.
+    """
     if not filtered_news:
         return f"No relevant news found for '{query}' in the energy market context."
 
-    general_news = [item for item in filtered_news if not item.get('is_company_news')]
-    company_news = [item for item in filtered_news if item.get('is_company_news')]
-
-    summaries = [item.get('analysis', '') for item in filtered_news]
+    # Combine the analyses from each news item for context
+    summaries = []
+    for item in filtered_news:
+        summaries.append(f"Title: {item.get('title', 'No title')}\nAnalysis: {item.get('analysis', '')}\n")
     combined_summary = "\n\n".join(summaries)
 
     prompt = f"""
-    Based on the following summaries of recent news articles and company announcements related to '{query}' in the energy market:
+    Based on the following news analyses:
     {combined_summary}
+    
     Provide a comprehensive market summary that:
-    1. Highlights the current trends and developments related to {query} in the energy market.
-    2. Identifies any significant impacts or potential changes in the market.
-    3. Mentions any notable events or decisions affecting this area.
-    4. Compares and contrasts information from general news sources and energy company announcements.
-    5. Identifies any discrepancies or complementary information between general news and company-specific news.
-    Keep the summary focused on factual information derived from the news articles and company announcements, without adding speculation or personal opinions.
-    Organize the summary into clear sections with appropriate subheadings.
+    - Highlights current trends related to {query} in the energy market.
+    - Identifies key insights and potential market impacts.
+    - Organizes the information into clearly defined sections.
     """
-
     try:
         response = model.generate_content(prompt)
         return response.text.strip()
     except Exception as e:
-        logging.error(f"Error generating market summary: {e}")
-        return f"Unable to generate market summary for '{query}' due to an error."
+        logging.error(f"Error generating market summary using API: {e}. Falling back to local summarization.")
+        return local_summarize(combined_summary)
 
 @app.route('/')
 def index():
     return render_template('index.html')
 
+# Add error handling decorator
+def handle_errors(f):
+    @wraps(f)
+    def wrapper(*args, **kwargs):
+        try:
+            return f(*args, **kwargs)
+        except Exception as e:
+            logging.error(f"Error in {f.__name__}: {str(e)}", exc_info=True)
+            return jsonify({'error': str(e)}), 500
+    return wrapper
+
 @app.route('/query', methods=['POST'])
+@handle_errors
 def query():
     data = request.json
+    if not data:
+        return jsonify({'error': 'No data provided'}), 400
+    
     query = data.get('query')
     role = data.get('role')
     file_id = data.get('file_id')
-    news_context = data.get('newsContext')
+    
+    if not query:
+        return jsonify({'error': 'No query provided'}), 400
+    if not role:
+        return jsonify({'error': 'No role provided'}), 400
 
     try:
-        if news_context:
-            # Process query with news context
-            prompt = f"""
-            You are an AI News Analyst specializing in the energy market. Use the following news context and your general knowledge to answer the query.
-
-            News Context:
-            Market Summary: {news_context.get('market_summary', 'No market summary available.')}
-
-            Top Articles:
-            {' '.join([f"- {article['title']}: {article['summary']}" for article in news_context.get('top_articles', [])])}
-
-            Query: {query}
-
-            Provide a comprehensive answer that:
-            1. Directly addresses the query using information from the news context.
-            2. Incorporates relevant general knowledge about the energy market.
-            3. Highlights any connections or insights between the query and the recent news.
-            4. Offers a balanced perspective, considering both general news and company-specific announcements.
-            5. Suggests potential implications or future trends based on the available information.
-
-            Format your response with clear headings and bullet points where appropriate.
-            """
-            response = model.generate_content(prompt)
-            return jsonify({'response': response.text})
-        else:
-            # Handle regular queries as before
-            response = process_query(query, role, file_id)
-            return jsonify({'response': response})
+        response = process_query(query, role, file_id)
+        return jsonify({'response': response})
     except Exception as e:
-        logging.error(f"Error in /query route: {str(e)}", exc_info=True)
+        logging.error(f"Error processing query: {str(e)}", exc_info=True)
         return jsonify({'error': str(e)}), 500
 
 @app.route('/upload', methods=['POST'])
+@handle_errors
 def upload_file():
     if 'file' not in request.files:
         return jsonify({'error': 'No file part'}), 400
+    
     file = request.files['file']
     if file.filename == '':
         return jsonify({'error': 'No selected file'}), 400
-    if file and allowed_file(file.filename):
-        filename = secure_filename(file.filename)
-
-        try:
-            file_content = file.read()
-            logging.info(f"File uploaded successfully: {filename}")
-            extracted_text = process_document(file_content, filename)
-            text_chunks = get_text_chunks(extracted_text)
-            analysis = analyze_document(extracted_text)
-
-            file_id = len(files_storage) + 1
-            files_storage[file_id] = {
-                'filename': filename,
-                'file_data': base64.b64encode(file_content).decode('utf-8'),
-                'analysis': analysis
-            }
-
-            create_vector_store(text_chunks, file_id)
-
-            logging.info(f"File processing completed and saved to in-memory storage with ID: {file_id}")
+    
+    if not allowed_file(file.filename):
+        return jsonify({'error': 'Invalid file type'}), 400
 
-            return jsonify({'file_id': file_id, 'analysis': analysis})
-        except Exception as e:
-            logging.error(f'Error processing file: {str(e)}', exc_info=True)
-            return jsonify({'error': str(e)}), 500
+    try:
+        file_content = file.read()
+        filename = secure_filename(file.filename)
+        
+        # Process the file
+        extracted_text = process_document(file_content, filename)
+        text_chunks = get_text_chunks(extracted_text)
+        analysis = analyze_document(extracted_text)
+
+        # Generate file ID and store
+        file_id = len(files_storage) + 1
+        files_storage[file_id] = {
+            'filename': filename,
+            'file_data': base64.b64encode(file_content).decode('utf-8'),
+            'analysis': analysis
+        }
+
+        # Create vector store
+        create_vector_store(text_chunks, file_id)
 
-    return jsonify({'error': 'Invalid file type'}), 400
+        return jsonify({
+            'file_id': file_id,
+            'analysis': analysis,
+            'message': 'File processed successfully'
+        })
+    except Exception as e:
+        logging.error(f"Error processing file: {str(e)}", exc_info=True)
+        return jsonify({'error': str(e)}), 500
 
 @app.route('/plot', methods=['POST'])
 def plot():
@@ -498,36 +636,101 @@ def process_dataframe_query(df, query):
 
 @app.route('/fetch_news', methods=['POST'])
 def fetch_news():
-    data = request.json
-    query = data.get('query')
     try:
-        all_articles = get_energy_news(query)
-        company_news = get_company_news()
-        filtered_news = filter_and_analyze_news(query, all_articles, company_news)
-        market_summary = generate_market_summary(query, filtered_news)
-
-        # Prepare the top 10 articles with summaries
-        top_articles = []
-        for article in filtered_news[:10]:
-            summary = article.get('analysis', '').split('\n\n')[0]  # Get the first paragraph of the analysis as summary
-            top_articles.append({
-                'title': article.get('title', 'No title'),
-                'url': article.get('link', '#'),
-                'source': article.get('source', 'Unknown'),
-                'summary': summary,
-                'is_company_news': article.get('is_company_news', False)
+        data = request.json
+        query = data.get('query', '')
+        
+        # Fetch articles from NewsData.io
+        articles = get_energy_news(query)
+        
+        if not articles:
+            return jsonify({
+                'status': 'error',
+                'message': 'No articles found',
+                'articles': [],
+                'summary': f"No relevant news found for '{query}'"
             })
 
+        # Analyze articles
+        analyzed_articles = []
+        for article in articles:
+            analysis = robust_analyze_news_item(article, query)
+            if analysis:
+                analyzed_articles.append(analysis)
+
+        if not analyzed_articles:
+            return jsonify({
+                'status': 'error',
+                'message': 'No articles could be analyzed',
+                'articles': [],
+                'summary': f"Could not analyze any articles for '{query}'"
+            })
+
+        # Generate market summary
+        summary_prompt = f"""
+        Based on the following analyzed news articles about "{query}":
+        
+        {' '.join([f"Article: {a['title']}\nAnalysis: {a['analysis']}\n\n" for a in analyzed_articles[:5]])}
+        
+        Provide a comprehensive market summary that:
+        1. Highlights the main trends and developments
+        2. Identifies potential market impacts
+        3. Suggests key takeaways for stakeholders
+        """
+
+        try:
+            summary_response = model.generate_content(summary_prompt)
+            market_summary = summary_response.text.strip()
+        except Exception as e:
+            logging.error(f"Error generating market summary: {str(e)}")
+            market_summary = "Unable to generate market summary due to an error."
+
         return jsonify({
-            'top_articles': top_articles,
-            'market_summary': market_summary,
-            'full_analysis': filtered_news
+            'status': 'success',
+            'articles': analyzed_articles,
+            'summary': market_summary
         })
+
     except Exception as e:
-        logging.error(f"Error in fetch_news route: {str(e)}", exc_info=True)
-        return jsonify({'error': str(e)}), 500
+        logging.error(f"Error in fetch_news route: {str(e)}")
+        return jsonify({
+            'status': 'error',
+            'message': str(e),
+            'articles': [],
+            'summary': "An error occurred while processing the news request."
+        }), 500
+
+# Add health check endpoint
+@app.route('/health', methods=['GET'])
+def health_check():
+    return jsonify({'status': 'healthy', 'api_key_configured': bool(google_api_key)})
+
+# Ensure all required environment variables are set
+def check_environment():
+    required_vars = ['GOOGLE_API_KEY']
+    missing_vars = [var for var in required_vars if not os.getenv(var)]
+    if missing_vars:
+        raise EnvironmentError(f"Missing required environment variables: {', '.join(missing_vars)}")
 
 if __name__ == '__main__':
-    # Use the PORT environment variable if it's set, otherwise default to 7860
-    port = int(os.environ.get('PORT', 7860))
-    app.run(host='0.0.0.0', port=port)
+    try:
+        # Check environment variables
+        check_environment()
+        
+        # Configure logging
+        logging.basicConfig(
+            level=logging.INFO,
+            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+        )
+        
+        # Initialize Google AI
+        if not google_api_key:
+            raise ValueError("GOOGLE_API_KEY not configured")
+        genai.configure(api_key=google_api_key)
+        
+        # Start server
+        port = int(os.environ.get('PORT', 7860))
+        app.run(host='0.0.0.0', port=port, debug=True)
+    except Exception as e:
+        logging.error(f"Failed to start server: {str(e)}", exc_info=True)
+        raise