fetch_news.py

import requests
from datetime import datetime, timedelta

def fetch_news_items():
    """
    Fetch the main TradingView news feed for XAUUSD and DXY,
    return it as a Python dictionary (JSON).
    """
    url = (
        "https://news-mediator.tradingview.com/news-flow/v2/news"
        "?filter=lang%3Aen&filter=symbol%3AOANDA%3AXAUUSD%2CTVC%3ADXY"
        "&client=screener&streaming=true"
    )
    headers = {
        "User-Agent": (
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
            "AppleWebKit/537.36 (KHTML, like Gecko) "
            "Chrome/138.0.0.0 Safari/537.36"
        )
    }
    response = requests.get(url, headers=headers)
    response.raise_for_status()
    return response.json()


def fetch_story_by_id(story_id):
    """
    Given a story ID from the news feed, fetch its full content JSON
    (shortDescription, astDescription, etc.) from TradingView.
    """
    base_url = "https://news-headlines.tradingview.com/v3/story"
    params = {
        "id": story_id,
        "lang": "en"
    }
    headers = {
        "User-Agent": (
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
            "AppleWebKit/537.36 (KHTML, like Gecko) "
            "Chrome/138.0.0.0 Safari/537.36"
        )
    }
    response = requests.get(base_url, headers=headers, params=params)
    response.raise_for_status()
    return response.json()


def parse_ast_node(node):
    """
    Recursively parse an AST node from the 'astDescription' field:
      - If it's a string, just return it.
      - If it's a dictionary with a known 'type', handle it.
      - Otherwise, parse children recursively.
    """
    if isinstance(node, str):
        # Direct text
        return node

    if isinstance(node, dict):
        node_type = node.get("type", "")
        children = node.get("children", [])

        if node_type == "root":
            # 'root' typically contains multiple children; parse them all
            return "".join(parse_ast_node(child) for child in children)

        elif node_type == "p":
            # Paragraph node: parse children, add a newline
            paragraph_text = "".join(parse_ast_node(child) for child in children)
            return paragraph_text + "\n"

        elif node_type == "url":
            # Format a hyperlink: [text](url)
            params = node.get("params", {})
            link_text = params.get("linkText", "")
            url = params.get("url", "")
            return f"{link_text} ({url})"

        elif node_type == "symbol":
            # Format a symbol mention
            params = node.get("params", {})
            symbol = params.get("symbol", "")
            text = params.get("text", "")
            return text if text else symbol

        else:
            # Unknown or unhandled node type; parse children anyway
            return "".join(parse_ast_node(child) for child in children)

    # If not string or dict (e.g., None?), return empty string
    return ""


def extract_full_content_from_story(story_data):
    """
    Extract the full textual content from a story, handling both
    shortDescription and the more detailed astDescription if available.
    """
    # Try to get the AST description first (more detailed)
    ast_description = story_data.get("astDescription")
    if ast_description:
        return parse_ast_node(ast_description)

    # Fall back to short description if AST not available
    return story_data.get("shortDescription", "No content available")


def get_recent_news_items(hours=100):
    """
    Fetch recent news items within the specified number of hours
    and return detailed information including publish time and full content.
    """
    # Get current time and calculate the threshold
    now_utc = datetime.utcnow()
    time_threshold = now_utc - timedelta(hours=hours)

    # Fetch news data
    news_data = fetch_news_items()
    items = news_data.get("items", [])
    detailed_news = []

    for item in items:
        item_id = item.get("id", "")
        published_ts = item.get("published")

        if not item_id or not published_ts:
            continue

        published_dt = datetime.utcfromtimestamp(published_ts)
        if published_dt >= time_threshold:
            # For each qualifying news item, fetch its full story content
            try:
                story_data = fetch_story_by_id(item_id)

                # Create a detailed news item
                detailed_item = {
                    "id": item_id,
                    "title": item.get("title", ""),
                    "source": item.get("source", {}).get("name", ""),
                    "published_time": published_dt.strftime("%Y-%m-%d %H:%M:%S UTC"),
                    "timestamp": published_ts,
                    "content": extract_full_content_from_story(story_data)
                }
                detailed_news.append(detailed_item)
            except Exception as e:
                print(f"Error fetching details for story {item_id}: {e}")

    # Sort by timestamp descending (newest first)
    detailed_news.sort(key=lambda x: x["timestamp"], reverse=True)
    return detailed_news


def build_news_summary(news_items):
    """
    Build a comprehensive summary of news items, including title, source,
    publication time, and full content.
    """
    if not news_items:
        return "No recent news available."

    summaries = []
    for item in news_items:
        summary = (
            f"📰 {item['title']}\n"
            f"🔍 Source: {item['source']}\n"
            f"⏰ Published: {item['published_time']}\n"
            f"📝 Content:\n{item['content']}\n"
            f"{'=' * 50}"
        )
        summaries.append(summary)
        
    return "\n\n".join(summaries)


def get_formatted_news_summary(hours=24):
    """
    Convenience function to get a formatted news summary for the specified time period.
    """
    news_items = get_recent_news_items(hours)
    return build_news_summary(news_items)


# Example usage:
if __name__ == "__main__":
    print("Fetching recent gold and DXY news...")
    news_summary = get_formatted_news_summary(48)  # Get news from the last 48 hours
    print(news_summary)