app/post_process.py

#!/usr/bin/env python3
"""
Post-process Polygraph newsletter HTML:
1. Convert Polymarket links to Dub tracking links
2. Minify HTML for Gmail compatibility (removes @font-face style tags, whitespace)

Usage: python post_process.py <input_html_file> <dub_api_key> [output_html_file]

IMPORTANT: This script uses conservative minification to preserve email formatting.
It does NOT remove:
- role="none" attributes (needed for table alignment in Gmail)
- Quoted attribute values (some email clients require them)
- font-style properties (even non-standard ones like Bold/Regular)
- Style tags (except those containing @font-face rules)
"""

import os
import re
import sys
import html
import requests
from urllib.parse import urlparse


# =============================================================================
# DUB LINK CONVERSION
# =============================================================================

def create_dub_tracking_link(url: str, title: str, api_key: str = None, tags: list = None) -> str:
    """Create Dub tracking link with multiple tags."""
    if not api_key:
        print(f"      Warning: Dub API key not provided, using original URL")
        return url

    try:
        if tags is None:
            tags = ["polygraph"]

        payload = {
            "url": url,
            "workspaceId": "ws_cm7dm89q90000qmskmss62vla",
            "title": title[:50] + "..." if len(title) > 50 else title,
            "comments": "Polymarket daily email",
            "trackConversion": True
        }

        if tags:
            payload["tagNames"] = tags

        response = requests.post(
            "https://api.dub.co/links",
            headers={
                "Authorization": f"Bearer {api_key}",
                "Content-Type": "application/json"
            },
            json=payload,
            timeout=10
        )

        if response.status_code in [200, 201]:
            data = response.json()
            short_link = data.get('shortLink') or data.get('shortUrl') or data.get('link')

            if short_link:
                if not short_link.startswith('http'):
                    print(f"      Warning: Invalid dub link format: {short_link}")
                    return url

                if 'polymarket.com' in short_link:
                    print(f"      Warning: Dub returned original URL")
                    return url

                if 'poly.market' not in short_link and 'dub.co' not in short_link and 'dub.sh' not in short_link:
                    print(f"      Warning: Unexpected domain: {short_link}")
                    return url

                print(f"      + {short_link}")
                return short_link
            else:
                print(f"      Warning: No short link in response")
                return url
        else:
            print(f"      Warning: Dub API error {response.status_code}")

        return url

    except Exception as e:
        print(f"      Warning: Dub exception: {str(e)}")
        return url


def extract_market_title(url: str) -> str:
    """Extract a title from the Polymarket URL."""
    path = urlparse(url).path
    parts = path.strip('/').split('/')

    if parts:
        title = parts[-1].replace('-', ' ').title()
        return title[:50]

    return "Polymarket Market"


def convert_polymarket_links(html_content: str, api_key: str = None) -> str:
    """Convert all Polymarket links in HTML to Dub tracking links."""
    if not api_key:
        print("  Skipping link conversion: No Dub API key provided")
        return html_content
    
    pattern = r'href=["\']([^"\']*polymarket\.com[^"\']*)["\']'
    matches = list(re.finditer(pattern, html_content))

    if not matches:
        print("  No Polymarket links found")
        return html_content

    print(f"  Found {len(matches)} Polymarket link(s) to convert")
    converted_count = 0

    for match in reversed(matches):
        original_url = match.group(1)
        decoded_url = html.unescape(original_url)
        title = extract_market_title(decoded_url)
        dub_url = create_dub_tracking_link(decoded_url, title, api_key=api_key, tags=["polygraph"])

        if dub_url != decoded_url:
            html_content = html_content[:match.start(1)] + dub_url + html_content[match.end(1):]
            converted_count += 1

    print(f"  Converted {converted_count} link(s) to Dub tracking links")
    return html_content


# =============================================================================
# HTML MINIFICATION (Conservative - preserves email formatting)
# =============================================================================

def remove_font_face_styles(html_content: str) -> str:
    """Remove <style> blocks containing @font-face rules from anywhere in the document.
    Email clients don't support @font-face, so these should be removed."""
    original_content = html_content
    
    # Find and remove style tags containing @font-face
    pattern = r'<style[^>]*>.*?@font-face.*?</style>'
    matches = list(re.finditer(pattern, html_content, flags=re.DOTALL | re.IGNORECASE))
    
    if matches:
        print(f"  Found {len(matches)} style tag(s) containing @font-face")
        # Remove from end to start to preserve indices
        for match in reversed(matches):
            html_content = html_content[:match.start()] + html_content[match.end():]
            print(f"    Removed @font-face style tag")
    else:
        print(f"  No @font-face style tags found")
    
    return html_content


def minify_whitespace(html_content: str) -> str:
    """Remove unnecessary whitespace while preserving structure.

    IMPORTANT: This is conservative - it only removes:
    - Blank lines
    - Leading/trailing whitespace on lines
    - Whitespace between tags

    It does NOT modify attribute values or remove attributes.
    """
    # Remove blank lines and strip each line, join without newlines
    lines = html_content.split('\n')
    lines = [line.strip() for line in lines if line.strip()]
    html_content = ''.join(lines)

    # Remove whitespace between tags (safe - doesn't affect content)
    html_content = re.sub(r'>\s+<', '><', html_content)

    return html_content


def fix_header_centering(html_content: str) -> str:
    """Ensure header container has margin:0 auto for centering."""
    html_content = re.sub(
        r'<div class="container" style="max-width:500px;\s*padding:0 16px;\s*text-align:center"',
        '<div class="container" style="max-width:500px; margin:0 auto; padding:0 16px; text-align:center"',
        html_content
    )
    return html_content


def fix_story_box_headlines(html_content: str) -> str:
    """Add margin-top:0 to story box h4 elements to prevent empty line above headlines."""
    # Match h4 in story boxes that have margin-bottom:8px but no margin-top:0
    def fix_h4(match):
        full = match.group(0)
        if 'margin-top:0' in full:
            return full
        # Insert margin-top:0 after margin-bottom:8px
        return full.replace("margin-bottom:8px'", "margin-bottom:8px; margin-top:0'")

    html_content = re.sub(r"<h4 style='[^']*margin-bottom:8px'[^>]*>", fix_h4, html_content)
    return html_content


def fix_unsubscribe_link(html_content: str) -> str:
    """Fix unsubscribe link href to use Customer.io merge tag.
    
    Finds links containing "unsubscribe here" text and updates their href
    from "#" (or empty) to {% manage_subscription_preferences_url %}.
    """
    modified = False
    
    # Pattern to match <a> tag with href="#" followed by "unsubscribe here" text
    # Matches: <a href="#" ...>...unsubscribe here...</a>
    # This pattern captures the full link tag including attributes and closing tag
    pattern = r'(<a[^>]*href=["\']#["\'][^>]*>.*?unsubscribe\s+here.*?</a>)'
    
    def replace_href(match):
        link_tag = match.group(0)
        # Replace href="#" with href="{% manage_subscription_preferences_url %}"
        updated_link = re.sub(
            r'href=["\']#["\']',
            'href="{% manage_subscription_preferences_url %}"',
            link_tag,
            flags=re.IGNORECASE
        )
        return updated_link
    
    if re.search(pattern, html_content, re.IGNORECASE | re.DOTALL):
        html_content = re.sub(pattern, replace_href, html_content, flags=re.IGNORECASE | re.DOTALL)
        modified = True
        print("  Fixed unsubscribe link href to {% manage_subscription_preferences_url %}")
    
    # Also handle case where just "here" is the link text (after "unsubscribe" text)
    # Pattern: <a href="#">here</a> when preceded by "unsubscribe" text
    pattern2 = r'(unsubscribe\s+)(<a[^>]*href=["\']#["\'][^>]*>.*?here.*?</a>)'
    def replace_href_here_only(match):
        before_text = match.group(1)
        link_tag = match.group(2)
        # Replace href="#" with href="{% manage_subscription_preferences_url %}"
        updated_link = re.sub(
            r'href=["\']#["\']',
            'href="{% manage_subscription_preferences_url %}"',
            link_tag,
            flags=re.IGNORECASE
        )
        return before_text + updated_link
    
    if re.search(pattern2, html_content, re.IGNORECASE | re.DOTALL):
        html_content = re.sub(pattern2, replace_href_here_only, html_content, flags=re.IGNORECASE | re.DOTALL)
        modified = True
        print("  Fixed unsubscribe 'here' link href to {% manage_subscription_preferences_url %}")
    
    if not modified:
        print(f"  No unsubscribe links found that need href fixing")
    
    return html_content


def fix_cloudinary_image_transformations(html_content: str) -> str:
    """Add Cloudinary transformations to market images that are missing them.
    
    This ensures all market images (48px square images) have proper transformations
    for cropping, while preserving aspect ratio for footer and top news images.
    """
    modified = False
    fixed_count = 0
    
    # Pattern to match img tags with Cloudinary URLs
    # Matches: <img ... src="https://res.cloudinary.com/.../upload/v123/..." ...>
    pattern = r'(<img[^>]*src=["\'])(https://res\.cloudinary\.com/[^"\']+)(["\'][^>]*>)'
    
    def add_transformations(match):
        nonlocal modified, fixed_count
        prefix = match.group(1)
        url = match.group(2)
        suffix = match.group(3)
        full_tag = match.group(0)
        
        # Check if URL already has transformations
        has_transformations = "/upload/w_" in url or "/upload/c_" in url or "/upload/h_" in url
        
        if has_transformations:
            return full_tag  # Already has transformations, skip
        
        # Check if this is a market image (48px width) vs footer/top news
        # Market images have width="48" or style containing "width:48px"
        is_market_image = False
        is_footer_or_top_news = False
        
        # Check for width="48" attribute
        if 'width="48"' in full_tag or "width='48'" in full_tag:
            is_market_image = True
        
        # Check for style with width:48px
        if re.search(r'width\s*:\s*48px', full_tag, re.IGNORECASE):
            is_market_image = True
        
        # Check if it's a footer image (alt="Polymarket" or in footer section)
        if 'alt="Polymarket"' in full_tag or 'alt=\'Polymarket\'' in full_tag:
            is_footer_or_top_news = True
        
        # Check if it's a top news image (inside top_news_box or alt="Top news image")
        if 'alt="Top news image"' in full_tag or 'alt=\'Top news image\'' in full_tag:
            is_footer_or_top_news = True
        
        # Check if it's a last word image (inside last_word_box or alt="Last word image")
        if 'alt="Last word image"' in full_tag or 'alt=\'Last word image\'' in full_tag:
            is_footer_or_top_news = True
        
        if not is_footer_or_top_news:
            # Check if it's inside a top_news_box or last_word_box by looking for the class in nearby HTML
            img_pos = html_content.find(full_tag)
            if img_pos != -1:
                # Check if there's a top_news_box or last_word_box before this img tag (within 1000 chars)
                before_img = html_content[max(0, img_pos - 1000):img_pos]
                # Look for top_news_box or last_word_box class (handle both quoted and unquoted, with/without spaces)
                if re.search(r'class\s*=\s*["\']?[^"\'>]*(top_news_box|last_word_box)', before_img, re.IGNORECASE):
                    is_footer_or_top_news = True
        
        # Only add transformations to market images (48px square images)
        if is_market_image and not is_footer_or_top_news:
            # Extract target size from width attribute or style
            target_size = 48  # Default for market images
            
            # Try to extract from width attribute
            width_match = re.search(r'width=["\'](\d+)["\']', full_tag, re.IGNORECASE)
            if width_match:
                target_size = int(width_match.group(1))
            else:
                # Try to extract from style
                style_match = re.search(r'width\s*:\s*(\d+)px', full_tag, re.IGNORECASE)
                if style_match:
                    target_size = int(style_match.group(1))
            
            # Add transformations to the URL
            transformation = f"w_{target_size},h_{target_size},c_fill,g_center,q_auto,f_auto"
            
            # Insert transformations into the URL
            if "/upload/" in url:
                parts = url.split("/upload/", 1)
                if len(parts) == 2:
                    path_part = parts[1]
                    new_url = f"{parts[0]}/upload/{transformation}/{path_part}"
                    modified = True
                    fixed_count += 1
                    return f"{prefix}{new_url}{suffix}"
        
        return full_tag  # No changes needed
    
    # Process all img tags
    html_content = re.sub(pattern, add_transformations, html_content, flags=re.IGNORECASE)
    
    if modified:
        print(f"  Fixed {fixed_count} Cloudinary image URL(s) by adding transformations")
    else:
        print(f"  No Cloudinary images found that need transformation fixes")
    
    return html_content


def fix_section_header_alignment(html_content: str) -> str:
    """Restore section header tables to the simplest, most compatible 2-column layout.
    
    Based on analysis of working emails (original_msg.eml), the most reliable pattern is:
    - Simple 2-column table with width="100%"
    - Second cell has align="right" attribute
    - NO table-layout:fixed
    - NO explicit widths on cells
    - NO spacer cells or nested tables
    
    This matches the original template structure that worked on mobile.
    """
    from bs4 import BeautifulSoup
    
    soup = BeautifulSoup(html_content, "html.parser")
    modified = False
    
    wrap_h_divs = soup.find_all("div", class_="wrap_h")
    for wrap_h in wrap_h_divs:
        tables = wrap_h.find_all("table", role="presentation")
        for table in tables:
            # Identify candidate header table: exactly one row with 2 tds
            tr = table.find("tr")
            if not tr:
                continue
            tds = tr.find_all("td", recursive=False)
            
            # Must have exactly 2 columns
            if len(tds) != 2:
                continue
            
            first_td, second_td = tds[0], tds[1]
            
            # Ensure header table is full width via attribute
            if table.get("width") != "100%":
                table["width"] = "100%"
                modified = True
            
            # Strip table-layout from table style if present
            table_style = table.get("style", "")
            if table_style and "table-layout" in table_style.lower():
                new_table_style = re.sub(r"table-layout\s*:\s*[^;]+;?\s*", "", table_style, flags=re.IGNORECASE).strip("; ")
                table["style"] = new_table_style
                modified = True
            
            # Remove width/min-width/white-space/padding-right constraints on both cells
            for td in (first_td, second_td):
                if td.get("width"):
                    del td["width"]
                    modified = True
                
                td_style = td.get("style", "")
                if td_style:
                    original_style = td_style
                    td_style = re.sub(r"width\s*:\s*[^;]+;?\s*", "", td_style, flags=re.IGNORECASE)
                    td_style = re.sub(r"min-width\s*:\s*[^;]+;?\s*", "", td_style, flags=re.IGNORECASE)
                    td_style = re.sub(r"white-space\s*:\s*[^;]+;?\s*", "", td_style, flags=re.IGNORECASE)
                    td_style = re.sub(r"padding-right\s*:\s*[^;]+;?\s*", "", td_style, flags=re.IGNORECASE)
                    td_style = td_style.strip("; ")
                    if td_style != original_style:
                        td["style"] = td_style
                        modified = True
            
            # Ensure second cell has align="right" attribute
            if second_td.get("align") != "right":
                second_td["align"] = "right"
                modified = True
            
            # Add text-align:right as fallback in inline style
            second_td_style = second_td.get("style", "")
            if "text-align" not in second_td_style.lower():
                # Clean up the style first - remove trailing semicolons and spaces
                second_td_style = second_td_style.strip().rstrip(";").strip()
                # Add text-align:right
                if second_td_style:
                    second_td_style += ";text-align:right"
                else:
                    second_td_style = "text-align:right"
                second_td["style"] = second_td_style
                modified = True
    
    if modified:
        print("  Restored section header tables to simple 2-column layout for mobile")
        return str(soup)
    
    print("  No section header tables found that need alignment fixes")
    return html_content


def add_link_spacing(html_content: str) -> str:
    """Add spaces before specific links to ensure proper spacing in email clients.
    
    This function works on the final HTML string and adds spaces before:
    - "Get Priority Access" links
    - "support@polymarket.com" links
    - "unsubscribe here" links
    """
    modified = False
    
    # Pattern 1: "Get Priority Access" link
    # Match: non-whitespace character immediately before <a> tag containing "Get Priority Access"
    pattern1 = r'([^\s&>])(<a[^>]*>.*?Get Priority Access)'
    if re.search(pattern1, html_content, re.IGNORECASE | re.DOTALL):
        html_content = re.sub(pattern1, r'\1 \2', html_content, flags=re.IGNORECASE | re.DOTALL)
        modified = True
        print(f"  Added space before 'Get Priority Access' link")
    
    # Pattern 2: "support@polymarket.com" link
    # Match: non-whitespace character immediately before <a> tag with mailto containing "support@polymarket.com"
    pattern2 = r'([^\s&>])(<a[^>]*mailto[^>]*>.*?support@polymarket\.com)'
    if re.search(pattern2, html_content, re.IGNORECASE | re.DOTALL):
        html_content = re.sub(pattern2, r'\1 \2', html_content, flags=re.IGNORECASE | re.DOTALL)
        modified = True
        print(f"  Added space before 'support@polymarket.com' link")
    
    # Pattern 3: "unsubscribe here" link
    # Match: non-whitespace character immediately before <a> tag containing "unsubscribe here" or just "here"
    pattern3 = r'([^\s&>])(<a[^>]*>.*?unsubscribe here)'
    if re.search(pattern3, html_content, re.IGNORECASE | re.DOTALL):
        html_content = re.sub(pattern3, r'\1 \2', html_content, flags=re.IGNORECASE | re.DOTALL)
        modified = True
        print(f"  Added space before 'unsubscribe here' link")
    
    # Also handle cases where "here" is the link text (for unsubscribe links)
    pattern4 = r'([^\s&>])(<a[^>]*href[^>]*manage_subscription[^>]*>.*?here)'
    if re.search(pattern4, html_content, re.IGNORECASE | re.DOTALL):
        html_content = re.sub(pattern4, r'\1 \2', html_content, flags=re.IGNORECASE | re.DOTALL)
        modified = True
        print(f"  Added space before 'unsubscribe here' link (manage_subscription)")
    
    if not modified:
        print(f"  No links found that need spacing")
    
    return html_content


def minify_html(html_content: str) -> str:
    """Apply conservative minification steps.

    This function is designed to reduce file size for Gmail while
    preserving all formatting attributes that email clients need.
    """
    original_size = len(html_content)
    print(f"  Starting minification (original size: {original_size:,} bytes)")

    # Step 1: Remove @font-face style tags (email clients don't support them)
    html_content = remove_font_face_styles(html_content)
    after_font_face = len(html_content)
    if after_font_face != original_size:
        print(f"  After removing @font-face: {after_font_face:,} bytes (saved {original_size - after_font_face:,} bytes)")

    # Step 2: Minify whitespace (conservative - only removes safe whitespace)
    html_content = minify_whitespace(html_content)
    after_whitespace = len(html_content)
    print(f"  After minifying whitespace: {after_whitespace:,} bytes")

    # Step 3: Fix header centering (add margin:0 auto if missing)
    html_content = fix_header_centering(html_content)
    if 'margin:0 auto' in html_content:
        print(f"  Fixed header centering (added margin:0 auto)")

    # Step 4: Fix story box headlines (add margin-top:0 to prevent empty line)
    html_content = fix_story_box_headlines(html_content)
    if "margin-top:0" in html_content:
        print(f"  Fixed story box headlines (added margin-top:0)")

    # Step 5: Fix unsubscribe link href (update from # to {% manage_subscription_preferences_url %})
    html_content = fix_unsubscribe_link(html_content)

    # Step 6: Fix Cloudinary image URLs (add transformations to market images)
    html_content = fix_cloudinary_image_transformations(html_content)

    # Step 7: Fix section header alignment for mobile
    html_content = fix_section_header_alignment(html_content)

    # Step 8: Add spaces before links (ensures proper spacing in email clients)
    html_content = add_link_spacing(html_content)

    final_size = len(html_content)
    total_saved = original_size - final_size
    percent_saved = (total_saved / original_size * 100) if original_size > 0 else 0
    print(f"  Final size: {final_size:,} bytes (saved {total_saved:,} bytes, {percent_saved:.1f}%)")

    return html_content


# =============================================================================
# VALIDATION
# =============================================================================

def validate_html(html_content: str) -> list:
    """Validate the HTML structure and return any errors."""
    errors = []

    if '<html' not in html_content:
        errors.append("Missing <html> tag")
    if '</html>' not in html_content:
        errors.append("Missing </html> closing tag")
    if '<head' not in html_content:
        errors.append("Missing <head> tag")
    if '<body' not in html_content:
        errors.append("Missing <body> tag")

    # Check for balanced tables
    table_opens = html_content.count('<table')
    table_closes = html_content.count('</table>')
    if table_opens != table_closes:
        errors.append(f"Unbalanced tables: {table_opens} opens, {table_closes} closes")

    # Check container structure
    if 'max-width:500px' not in html_content:
        errors.append("Container max-width missing")

    # Check that we preserved important attributes
    if 'role="none"' not in html_content:
        errors.append("Warning: role='none' attributes missing (may affect table alignment)")

    if 'cellpadding="0"' not in html_content:
        errors.append("Warning: quoted cellpadding attributes missing (may affect email clients)")

    return errors


# =============================================================================
# MAIN
# =============================================================================

def main():
    if len(sys.argv) < 3:
        print("Usage: python post_process.py <input_html_file> <dub_api_key> [output_html_file]")
        print("\nThis script:")
        print("  1. Converts Polymarket links to Dub tracking links")
        print("  2. Minifies HTML for Gmail compatibility (conservative)")
        print("\nArguments:")
        print("  input_html_file: Path to input HTML file")
        print("  dub_api_key: Dub API key for link tracking")
        print("  output_html_file: (Optional) Path to output file (default: input_final.html)")
        print("\nConservative minification preserves:")
        print("  - role='none' attributes (table alignment)")
        print("  - Quoted attribute values (email client compatibility)")
        print("  - font-style properties (even non-standard ones)")
        sys.exit(1)

    input_file = sys.argv[1]
    dub_api_key = sys.argv[2]

    # Default output: replace .html with _final.html
    if len(sys.argv) > 3:
        output_file = sys.argv[3]
    else:
        output_file = input_file.replace('.html', '_final.html')

    # Read input
    print(f"Reading {input_file}...")
    with open(input_file, 'r', encoding='utf-8') as f:
        html_content = f.read()

    original_size = len(html_content)
    print(f"  Original size: {original_size:,} bytes\n")

    # Step 1: Convert Polymarket links to Dub tracking links
    print("Step 1: Converting Polymarket links to Dub tracking links...")
    html_content = convert_polymarket_links(html_content, api_key=dub_api_key)
    print()

    # Step 2: Minify HTML (conservative)
    print("Step 2: Minifying HTML for Gmail (conservative)...")
    html_content = minify_html(html_content)
    print()

    # Validate
    print("Validating HTML structure...")
    errors = validate_html(html_content)
    warnings = [e for e in errors if e.startswith("Warning:")]
    real_errors = [e for e in errors if not e.startswith("Warning:")]

    if real_errors:
        print("  ERRORS:")
        for e in real_errors:
            print(f"    - {e}")
    if warnings:
        print("  Warnings:")
        for w in warnings:
            print(f"    - {w}")
    if not errors:
        print("  Structure OK")

    # Check Gmail compatibility
    final_size = len(html_content)
    gmail_limit = 102000
    if final_size < gmail_limit:
        print(f"  Gmail: OK ({final_size:,} < {gmail_limit:,} bytes)")
    else:
        print(f"  Gmail: WARNING - may be clipped ({final_size:,} >= {gmail_limit:,} bytes)")

    print()

    # Write output
    print(f"Writing to {output_file}...")
    with open(output_file, 'w', encoding='utf-8') as f:
        f.write(html_content)

    print(f"\nDone! Saved to {output_file}")
    print(f"  Original: {original_size:,} bytes")
    print(f"  Final: {final_size:,} bytes")
    print(f"  Saved: {original_size - final_size:,} bytes ({((original_size - final_size) / original_size * 100):.1f}%)")


if __name__ == "__main__":
    main()