The script

app/config.py

@@ -8,12 +8,12 @@ from pydantic_settings import BaseSettings
 class Settings(BaseSettings):
     """Central configuration for the HuggingFace backend."""
 
-    default_num_items: int = Field(10, env="POLYGRAPH_DEFAULT_NUM_ITEMS")
-    skip_dub: bool = Field(False, env="POLYGRAPH_SKIP_DUB")
-    template_root: str = Field("polygraph(email)/polygraph(email)", env="POLYGRAPH_TEMPLATE_DIR")
-    template_archive_dir: str = Field("template_archive", env="POLYGRAPH_TEMPLATE_ARCHIVE")
-    header_path: str = Field("header.png", env="POLYGRAPH_HEADER_PATH")
-    header_link: str = Field("https://polymarket.com", env="POLYGRAPH_HEADER_LINK")
+    default_num_items: int = Field(10, env="KALSHI_DEFAULT_NUM_ITEMS")
+    skip_dub: bool = Field(False, env="KALSHI_SKIP_DUB")
+    template_root: str = Field("kalshi(email)/kalshi(email)", env="KALSHI_TEMPLATE_DIR")
+    template_archive_dir: str = Field("template_archive", env="KALSHI_TEMPLATE_ARCHIVE")
+    header_path: str = Field("header.png", env="KALSHI_HEADER_PATH")
+    header_link: str = Field("https://kalshi.com", env="KALSHI_HEADER_LINK")
     supabase_url: Optional[str] = Field(None, env="SUPABASE_URL")
     supabase_service_key: Optional[str] = Field(None, env="SUPABASE_SERVICE_KEY")
     storage_prefix: str = Field("", env="STORAGE_PREFIX")

app/email3.py

@@ -1,1869 +0,0 @@
-#!/usr/bin/env python3
-"""
-Polymarket Data Collection Script
-Fetches and organizes market data, comments, and whale moves from Polymarket
-Similar to breaking.py but with multiple data sections
-"""
-
-import argparse
-import json
-import os
-import sys
-import re
-import base64
-import cloudinary
-import cloudinary.uploader
-from datetime import datetime, timedelta
-from typing import Dict, List, Any, Optional, Tuple
-from curl_cffi import requests
-from collections import defaultdict
-from copy import deepcopy
-from dotenv import load_dotenv
-
-# Load environment variables from .env file if it exists
-load_dotenv()
-
-# Placeholder data used when upstream data sources are unavailable.
-PLACEHOLDER_WHALE_MOVES = [
-    {
-        "title": "$441,975 position on No",
-        "market": "Xi Jinping out in 2025?",
-        "event": "Xi Jinping out in 2025?",
-        "slug": "xi-jinping-out-in-2025",
-        "amount": 441974.7783061075,
-        "user_name": "7d74",
-        "user_profile_url": "7d74",
-        "distinct_positions": 2,
-        "timestamp": datetime.utcnow().isoformat(),
-        "market_image": "",  # Will be fetched
-        "user_image": "https://i.ibb.co/23VYpRcK/polywhale.png"
-    },
-    {
-        "title": "$160,910 position on Yes",
-        "market": "Luiz Inácio Lula da Silva",
-        "event": "Brazil Presidential Election",
-        "slug": "brazil-presidential-election",
-        "amount": 160909.54283122497,
-        "user_name": "UnknwnFnd",
-        "user_profile_url": "UnknwnFnd",
-        "distinct_positions": 4,
-        "timestamp": datetime.utcnow().isoformat(),
-        "market_image": "",  # Will be fetched
-        "user_image": "https://i.ibb.co/23VYpRcK/polywhale.png"
-    },
-    {
-        "title": "$122,836 position on No",
-        "market": "Russia x Ukraine ceasefire by end of 2026?",
-        "event": "Russia x Ukraine ceasefire by end of 2026?",
-        "slug": "russia-x-ukraine-ceasefire-by-end-of-2026",
-        "amount": 122835.693405475,
-        "user_name": "Businio",
-        "user_profile_url": "Businio",
-        "distinct_positions": 1,
-        "timestamp": datetime.utcnow().isoformat(),
-        "market_image": "",  # Will be fetched
-        "user_image": "https://i.ibb.co/23VYpRcK/polywhale.png"
-    },
-]
-
-PLACEHOLDER_COMMENTS = [
-    {
-        "comment": "Seasonality alone doesn\u2019t explain the flow here. This market is really a referendum on how quickly energy prices ease and that path still looks rough.",
-        "author": "MacroRhino",
-        "market_title": "Will US inflation fall below 3% in 2025?",
-        "slug": "us-inflation-below-3-2025",
-        "likes": 42,
-        "createdAt": datetime.utcnow().isoformat(),
-        "weighted_score": 8.4,
-        "age_days": 1.1,
-        "hours_ago": 6,
-        "hotness": 9.5
-    },
-    {
-        "comment": "Desk chatter says the campaign is actually accelerating spend in WI + PA. I still like \u2018no\u2019 at 38 but you have to size for a very noisy news cycle.",
-        "author": "ElectionNerd",
-        "market_title": "Will Democrats win Wisconsin in 2024?",
-        "slug": "democrats-win-wisconsin-2024",
-        "likes": 57,
-        "createdAt": datetime.utcnow().isoformat(),
-        "weighted_score": 10.2,
-        "age_days": 0.6,
-        "hours_ago": 3,
-        "hotness": 14.1
-    },
-    {
-        "comment": "Markets are underpricing how quickly AI-capex translates into revenue. Smells like peak pessimism on the megacap multiples.",
-        "author": "GammaFlow",
-        "market_title": "Will NVDA finish 2025 above $1600?",
-        "slug": "nvidia-1600-2025",
-        "likes": 33,
-        "createdAt": datetime.utcnow().isoformat(),
-        "weighted_score": 7.1,
-        "age_days": 1.6,
-        "hours_ago": 12,
-        "hotness": 6.8
-    },
-]
-
-
-def _fresh_placeholder_whales() -> List[Dict[str, Any]]:
-    """Return whale placeholder data with refreshed timestamps."""
-    whales = deepcopy(PLACEHOLDER_WHALE_MOVES)
-    timestamp = datetime.utcnow().isoformat()
-    for entry in whales:
-        entry["timestamp"] = timestamp
-    return whales
-
-
-def _fresh_placeholder_comments() -> List[Dict[str, Any]]:
-    """Return comment placeholder data with refreshed timing metadata."""
-    now = datetime.utcnow()
-    comments = deepcopy(PLACEHOLDER_COMMENTS)
-    for idx, entry in enumerate(comments):
-        entry["createdAt"] = now.isoformat()
-        entry["age_days"] = round(0.5 + idx * 0.4, 2)
-        entry["hours_ago"] = round(idx * 3 + 1, 1)
-        entry["weighted_score"] = entry.get("weighted_score", 1.0) + idx
-        entry["hotness"] = entry.get("hotness", 5.0) + idx
-    return comments
-
-def upload_to_cloudinary(png_data: bytes, filename: str, date_folder: str, apply_transform: bool = True) -> str:
-    """Upload image to Cloudinary with folder organization"""
-    cloudinary_url = os.getenv('CLOUDINARY_URL')
-
-    if not cloudinary_url:
-        print(f"      No CLOUDINARY_URL found - using placeholder")
-        return "https://via.placeholder.com/48"
-
-    try:
-        # Convert png_data to base64
-        if isinstance(png_data, bytes):
-            # Detect image format from data
-            if png_data[:4] == b'\x89PNG':
-                mime_type = "image/png"
-            elif png_data[:2] == b'\xff\xd8':
-                mime_type = "image/jpeg"
-            else:
-                mime_type = "image/png"  # Default to PNG
-
-            image_base64 = base64.b64encode(png_data).decode('utf-8')
-            upload_data = f"data:{mime_type};base64,{image_base64}"
-        else:
-            upload_data = png_data
-
-        # Upload with folder organization
-        folder_path = f"polymarket/{date_folder}"
-
-        result = cloudinary.uploader.upload(
-            upload_data,
-            folder=folder_path
-        )
-
-        # Get the URL and optionally add transformations
-        if result and result.get('secure_url'):
-            image_url = result['secure_url']
-            print(f"      Uploaded to Cloudinary: {filename}")
-
-            # Only add transformation for market images (48x48)
-            if apply_transform and '/upload/' in image_url:
-                transformed_url = image_url.replace('/upload/', '/upload/w_48,h_48,c_fill,q_auto,f_auto/')
-                return transformed_url
-            return image_url
-        else:
-            return "https://via.placeholder.com/48"
-
-    except Exception as e:
-        print(f"      Cloudinary upload failed: {e}")
-        return "https://via.placeholder.com/48"
-
-
-def create_dub_tracking_link(url: str, title: str, tags: list = None, skip_dub: bool = False) -> str:
-    """Create Dub tracking link with multiple tags."""
-    # Skip if nodub flag is set
-    if skip_dub:
-        return url
-
-    api_key = os.getenv('DUB_API_KEY')
-
-    if not api_key:
-        print(f"      ⚠️  DUB_API_KEY not set, using original URL")
-        return url
-
-    try:
-        # Default tags if none provided
-        if tags is None:
-            tags = []
-
-        payload = {
-            "url": url,
-            "workspaceId": "ws_cm7dm89q90000qmskmss62vla",
-            "title": title[:50] + "..." if len(title) > 50 else title,
-            "comments": "Polymarket daily email",
-            "trackConversion": True
-        }
-
-        # Only add tags if they are provided and not empty
-        if tags:
-            payload["tagNames"] = tags
-
-        response = requests.post(
-            "https://api.dub.co/links",
-            headers={
-                "Authorization": f"Bearer {api_key}",
-                "Content-Type": "application/json"
-            },
-            json=payload,
-            timeout=10,
-            impersonate="chrome110"
-        )
-
-        if response.status_code in [200, 201]:
-            data = response.json()
-            # Debug: log key parts of the response (commented out - uncomment if needed)
-            # print(f"      DEBUG: Dub response contains: shortLink={data.get('shortLink')}, link={data.get('link')}, shortUrl={data.get('shortUrl')}")
-
-            # Try different possible field names for the short URL
-            short_link = data.get('shortLink') or data.get('shortUrl') or data.get('link')
-
-            if short_link:
-                # Validate the short link format
-                if not short_link.startswith('http'):
-                    print(f"      ⚠️  Invalid dub link format: {short_link}, using original URL")
-                    return url
-
-                # Check for expected short link domains
-                if 'polymarket.com' in short_link:
-                    # This shouldn't happen - it's returning the original URL
-                    print(f"      ⚠️  Dub returned original URL: {short_link}, API may have failed")
-                    return url
-
-                # poly.market is the CORRECT domain for shortened links
-                if 'poly.market' not in short_link and 'dub.co' not in short_link and 'dub.sh' not in short_link:
-                    print(f"      ⚠️  Unexpected domain in short link: {short_link}")
-                    return url
-
-                print(f"      ✓ Created tracking link: {short_link}")
-                return short_link
-            else:
-                print(f"      ⚠️  No short link in response: {data}")
-                return url
-        else:
-            print(f"      ⚠️  Dub API error {response.status_code}: {response.text[:100]}")
-
-        return url
-
-    except Exception as e:
-        print(f"      ⚠️  Dub exception: {str(e)}")
-        return url
-
-def calculate_time_remaining(end_date_str: str) -> str:
-    """Calculate time remaining until market closes"""
-    try:
-        end_date = datetime.fromisoformat(end_date_str.replace('Z', '+00:00'))
-        now = datetime.now(end_date.tzinfo)
-        delta = end_date - now
-        
-        if delta.days > 0:
-            return f"{delta.days}d"
-        elif delta.seconds > 3600:
-            hours = delta.seconds // 3600
-            return f"{hours}h"
-        else:
-            minutes = delta.seconds // 60
-            return f"{minutes}m"
-    except:
-        return "Soon"
-
-def calculate_time_since(created_date_str: str) -> str:
-    """Calculate time since market was created"""
-    try:
-        if not created_date_str:
-            return "Recently"
-            
-        # Handle different date formats
-        from dateutil import parser
-        created_date = parser.parse(created_date_str)
-        
-        # Make sure both dates are timezone aware
-        if created_date.tzinfo is None:
-            from datetime import timezone
-            created_date = created_date.replace(tzinfo=timezone.utc)
-        
-        now = datetime.now(created_date.tzinfo)
-        delta = now - created_date
-        
-        # More granular time display
-        total_seconds = delta.total_seconds()
-        
-        if delta.days > 30:
-            months = delta.days // 30
-            return f"{months}mo ago"
-        elif delta.days > 0:
-            if delta.days == 1:
-                return "1d ago"
-            return f"{delta.days}d ago"
-        elif total_seconds > 3600:
-            hours = int(total_seconds // 3600)
-            if hours == 1:
-                return "1h ago"
-            return f"{hours}h ago"
-        elif total_seconds > 60:
-            minutes = int(total_seconds // 60)
-            if minutes == 1:
-                return "1m ago"
-            return f"{minutes}m ago"
-        else:
-            return "Just now"
-    except Exception as e:
-        print(f"      Warning: Could not parse date '{created_date_str}': {e}")
-        return "Recently"
-
-class PolymarketEmailGenerator:
-    """Main class for generating Polymarket data emails"""
-
-    def __init__(self, num_items: int = 10, skip_dub: bool = False):
-        """Initialize with number of items per section"""
-        self.num_items = num_items
-        self.skip_dub = skip_dub
-        self.selected_markets = []
-        self.menu_items = {}
-        self.market_data_cache = {}
-        
-    def fetch_market_details(self, slug: str) -> Dict[str, Any]:
-        """Fetch detailed market information including image"""
-        try:
-            # Try to get from cache first
-            if slug in self.market_data_cache:
-                return self.market_data_cache[slug]
-            
-            # Fetch market details using slug parameter instead of path
-            response = requests.get(
-                "https://gamma-api.polymarket.com/events",
-                params={"slug": slug},
-                timeout=10,
-                impersonate="chrome110"
-            )
-            response.raise_for_status()
-            
-            # The API returns an array, get the first item
-            data = response.json()
-            if data and isinstance(data, list) and len(data) > 0:
-                market_data = data[0]
-                self.market_data_cache[slug] = market_data
-                return market_data
-            else:
-                return {}
-            
-        except Exception as e:
-            print(f"      Warning: Could not fetch details for {slug}: {e}")
-            return {}
-        
-    def fetch_ending_soon(self) -> List[Dict[str, Any]]:
-        """Fetch markets ending soon - ONLY non-sports, non-crypto markets"""
-        print("Fetching markets ending soon (no sports/crypto)...")
-
-        try:
-            from datetime import timezone
-            from dateutil import parser
-            now = datetime.now(timezone.utc)
-
-            all_markets = []
-
-            # Strategy 1: Get high volume markets (likely to be current) - paginated
-            # Fetch 400 total (4 pages of 100)
-            for offset in range(0, 900, 100):
-                response = requests.get(
-                    "https://gamma-api.polymarket.com/events",
-                    params={
-                        "order": "volume24hr",
-                        "ascending": "false",
-                        "limit": 100,
-                        "offset": offset
-                    },
-                    timeout=10,
-                    impersonate="chrome110"
-                )
-                if response.status_code == 200:
-                    page_data = response.json()
-                    all_markets.extend(page_data)
-                    # If we got fewer than 100, we've reached the end
-                    if len(page_data) < 100:
-                        break
-                else:
-                    break
-
-            # Deduplicate by slug
-            seen_slugs = set()
-            unique_markets = []
-            for market in all_markets:
-                slug = market.get('slug', '')
-                if slug and slug not in seen_slugs:
-                    seen_slugs.add(slug)
-                    unique_markets.append(market)
-
-            # Filter for non-sports, non-crypto markets only
-            valid_markets = []
-
-            # Sports keywords to EXCLUDE
-            sports_keywords = ['vs.', ' vs ', 'NFL', 'NBA', 'MLB', 'NHL', 'UFC', 'MMA',
-                             'Soccer', 'Football', 'Basketball', 'Baseball', 'Hockey',
-                             'Tennis', 'Golf', 'Boxing', 'Premier League', 'Champions League',
-                             'ucl-', 'mlb-', 'nba-', 'nfl-', 'nhl-', 'wnba-', 'Serie A',
-                             'La Liga', 'Bundesliga', 'Ligue 1', 'UEFA', 'FIFA', 'World Cup',
-                             'Cavaliers', 'Lakers', 'Warriors', 'Celtics', 'Knicks', 'Nets',
-                             'Yankees', 'Dodgers', 'Astros', 'Heisman', 'Davey O\'Brien',
-                             'Doak Walker', 'Biletnikoff', 'Award Winner', 'cfb-', 'ncaa',
-                             'Bowl Game', 'Championship Game', 'playoffs', 'tournament']
-
-            # Esports keywords to EXCLUDE
-            esports_keywords = ['LoL:', 'Dota', 'CS:GO', 'Valorant', 'Overwatch', 'Rocket League',
-                               'Fortnite', 'PUBG', 'Apex Legends', 'Rainbow Six', 'Call of Duty',
-                               'esports', 'e-sports', '(BO3)', '(BO5)', 'Gen.G', 'T1', 'TSM',
-                               'Team Liquid', 'Cloud9', 'FaZe', 'NaVi', 'Fnatic', 'G2',
-                               'Mobile Legends', 'MLBB', 'Honor of Kings', 'Arena of Valor',
-                               'League of Legends:', 'this week', 'StarCraft', 'Hearthstone', 'Overwatch League']
-
-            # Crypto/trading keywords to EXCLUDE - expanded for short-term
-            crypto_keywords = ['bitcoin', 'ethereum', 'btc', 'eth', 'solana', 'sol', 'xrp',
-                              'crypto', 'coin', 'token', 'above', 'below', 'hit',
-                              'multistrike', '4pm et', '8pm et', '12pm et', 'trading',
-                              'market cap', 'defi', 'nft', 'blockchain', '3:00pm', '3:15pm',
-                              '3:30pm', '3:45pm', 'price -', 'above ___', 'below ___',
-                              'price on october', 'price on november', 'price on december',
-                              'price on january', 'price on february', 'price on march',
-                              'what price will', 'binance', 'coinbase', 'doge', 'shib',
-                              'cardano', 'ada', 'bnb', 'polygon', 'matic', 'avalanche',
-                              'avax', 'polkadot', 'dot', 'chainlink', 'link', 'tweets']
-
-            # Weather keywords to EXCLUDE
-            weather_keywords = ['temperature', 'degrees', 'rain', 'snow', 'weather', 'storm',
-                                'hurricane', 'tornado', 'hotter', 'colder', 'warmest', 'coldest',
-                                'precipitation', 'humidity', 'forecast', 'climate']
-
-            for event in unique_markets:
-                # Skip if marked as ended or resolved
-                if event.get('ended', False) or event.get('resolved', False):
-                    continue
-
-                # Get end date
-                end_date_str = event.get("endDate") or event.get("closedTime") or event.get("closeTime", "")
-                if not end_date_str:
-                    continue
-
-                try:
-                    end_date = parser.parse(end_date_str)
-                    if end_date.tzinfo is None:
-                        end_date = end_date.replace(tzinfo=timezone.utc)
-
-                    delta = end_date - now
-
-                    # Skip if in the past or too far future (3 days max)
-                    if delta.total_seconds() <= 0 or delta.days > 3:
-                        continue
-
-                    title = event.get('title', '')
-                    slug = event.get('slug', '')
-                    title_lower = title.lower()
-                    slug_lower = slug.lower()
-
-                    # Skip if it's a sports market
-                    if any(keyword in title or keyword in title_lower or keyword in slug_lower
-                          for keyword in sports_keywords):
-                        continue
-
-                    # Skip if it's an esports market
-                    if any(keyword in title or keyword in title_lower or keyword in slug_lower
-                          for keyword in esports_keywords):
-                        continue
-
-                    # Skip if it's a crypto/trading market
-                    if any(keyword in title_lower or keyword in slug_lower
-                          for keyword in crypto_keywords):
-                        continue
-
-                    # Skip if it's a weather market
-                    if any(keyword in title_lower or keyword in slug_lower
-                          for keyword in weather_keywords):
-                        continue
-
-                    # Skip "Up or Down" markets
-                    if 'up or down' in title_lower:
-                        continue
-
-                    # Skip if market is closed (even if not marked as ended)
-                    if event.get('closed', False):
-                        continue
-
-                    # This is a valid non-sports, non-crypto market
-                    valid_markets.append({
-                        'event': event,
-                        'end_date': end_date,
-                        'hours_until': delta.total_seconds() / 3600
-                    })
-
-                except Exception as e:
-                    continue
-
-            # Sort by end date (soonest first)
-            valid_markets.sort(key=lambda x: x['end_date'])
-
-            # Format results
-            markets = []
-            for item in valid_markets[:15]:
-                event = item['event']
-
-                # Fetch detailed data for image
-                details = self.fetch_market_details(event.get("slug", ""))
-
-                markets.append({
-                    "title": event.get("title", "Unknown"),
-                    "slug": event.get("slug", ""),
-                    "closedTime": item['end_date'].isoformat(),
-                    "volume": float(event.get("volume", 0)),
-                    "liquidity": float(event.get("liquidity", 0)),
-                    "image": details.get("image", event.get("image", "")),
-                    "time_remaining": calculate_time_remaining(item['end_date'].isoformat())
-                })
-
-            print(f"   Found {len(valid_markets)} non-sports/crypto markets ending soon")
-            return markets
-
-        except Exception as e:
-            print(f"   Warning: API error: {e}")
-            return []
-    
-    def fetch_just_listed(self) -> List[Dict[str, Any]]:
-        """Fetch recently listed markets - ONLY non-sports, non-crypto"""
-        print("Fetching newly listed markets (no sports/crypto)...")
-
-        try:
-            from datetime import timezone
-            from dateutil import parser
-            now = datetime.now(timezone.utc)
-
-            # Use createdAt to get truly new markets (not just updated ones) - paginated
-            # Fetch 800 total (8 pages of 100)
-            events = []
-            for offset in range(0, 6000, 500):
-                response = requests.get(
-                    "https://gamma-api.polymarket.com/events",
-                    params={
-                        "order": "createdAt",
-                        "ascending": "false",  # Most recently created first
-                        "limit": 500,
-                        "offset": offset
-                    },
-                    timeout=10,
-                    impersonate="chrome110"
-                )
-                if response.status_code == 200:
-                    page_data = response.json()
-                    events.extend(page_data)
-                    # If we got fewer than 100, we've reached the end
-                    if len(page_data) < 100:
-                        break
-                else:
-                    response.raise_for_status()
-                    break
-            
-            # Count unique markets by slug
-            unique_slugs = set()
-            for event in events:
-                slug = event.get('slug', '')
-                if slug:
-                    unique_slugs.add(slug)
-            print(f"   Checked {len(unique_slugs)} unique markets")
-            
-            markets = []
-
-            # Sports keywords to EXCLUDE
-            sports_keywords = ['vs.', ' vs ', 'NFL', 'NBA', 'MLB', 'NHL', 'UFC', 'MMA',
-                             'Soccer', 'Football', 'Basketball', 'Baseball', 'Hockey',
-                             'Tennis', 'Golf', 'Boxing', 'Premier League', 'Champions League',
-                             'ucl-', 'mlb-', 'nba-', 'nfl-', 'nhl-', 'wnba-', 'Serie A',
-                             'La Liga', 'Bundesliga', 'Ligue 1', 'UEFA', 'FIFA', 'World Cup',
-                             'Cavaliers', 'Lakers', 'Warriors', 'Celtics', 'Knicks', 'Nets',
-                             'Yankees', 'Dodgers', 'Astros', 'Heisman', 'Davey O\'Brien',
-                             'Doak Walker', 'Biletnikoff', 'Award Winner', 'cfb-', 'ncaa',
-                             'Bowl Game', 'Championship Game', 'playoffs', 'tournament']
-
-            # Esports keywords to EXCLUDE
-            esports_keywords = ['LoL:', 'Dota', 'CS:GO', 'Valorant', 'Overwatch', 'Rocket League',
-                               'Fortnite', 'PUBG', 'Apex Legends', 'Rainbow Six', 'Call of Duty',
-                               'esports', 'e-sports', '(BO3)', '(BO5)', 'Gen.G', 'T1', 'TSM',
-                               'Team Liquid', 'Cloud9', 'FaZe', 'NaVi', 'Fnatic', 'G2',
-                               'Mobile Legends', 'MLBB', 'Honor of Kings', 'Arena of Valor',
-                               'League of Legends:', 'StarCraft', 'Hearthstone', 'Overwatch League']
-
-            # Crypto/trading keywords to EXCLUDE - expanded for short-term
-            crypto_keywords = ['bitcoin', 'ethereum', 'btc', 'eth', 'solana', 'sol', 'xrp',
-                              'crypto', 'coin', 'token', 'above', 'below', 'hit',
-                              'multistrike', '4pm et', '8pm et', '12pm et', 'trading',
-                              'market cap', 'defi', 'nft', 'blockchain', '3:00pm', '3:15pm',
-                              '3:30pm', '3:45pm', 'price -', 'above ___', 'below ___',
-                              'price on october', 'price on november', 'price on december',
-                              'price on january', 'price on february', 'price on march',
-                              'what price will', 'binance', 'coinbase', 'doge', 'shib',
-                              'cardano', 'ada', 'bnb', 'polygon', 'matic', 'avalanche',
-                              'avax', 'polkadot', 'dot', 'chainlink', 'link']
-
-            # Weather keywords to EXCLUDE
-            weather_keywords = ['temperature', 'degrees', 'rain', 'snow', 'weather', 'storm',
-                                'hurricane', 'tornado', 'hotter', 'colder', 'warmest', 'coldest',
-                                'precipitation', 'humidity', 'forecast', 'climate']
-
-            for event in events:
-                # Skip if closed, ended, or resolved
-                if event.get('closed', False) or event.get('ended', False) or event.get('resolved', False):
-                    continue
-
-                title = event.get('title', '')
-                slug = event.get('slug', '')
-                title_lower = title.lower()
-                slug_lower = slug.lower()
-
-                # Skip "Up or Down" markets
-                if 'up or down' in title_lower:
-                    continue
-
-                # Skip if it's a sports market
-                if any(keyword in title or keyword in title_lower or keyword in slug_lower
-                      for keyword in sports_keywords):
-                    continue
-
-                # Skip if it's an esports market
-                if any(keyword in title or keyword in title_lower or keyword in slug_lower
-                      for keyword in esports_keywords):
-                    continue
-
-                # Skip if it's a crypto/trading market
-                if any(keyword in title_lower or keyword in slug_lower
-                      for keyword in crypto_keywords):
-                    continue
-
-                # Skip if it's a weather market
-                if any(keyword in title_lower or keyword in slug_lower
-                      for keyword in weather_keywords):
-                    continue
-
-                # Check how recently it was created
-                created_at = event.get("createdAt", "")
-                if created_at:
-                    try:
-                        created_date = parser.parse(created_at)
-                        if created_date.tzinfo is None:
-                            created_date = created_date.replace(tzinfo=timezone.utc)
-
-                        # Skip if created more than 14 days ago (expanded window due to heavy filtering)
-                        days_old = (now - created_date).days
-                        if days_old > 14:
-                            continue
-                    except:
-                        pass
-
-                # Fetch detailed data for image
-                details = self.fetch_market_details(event.get("slug", ""))
-
-                markets.append({
-                    "title": event.get("title", "Unknown"),
-                    "slug": event.get("slug", ""),
-                    "createdAt": event.get("createdAt", ""),
-                    "volume": float(event.get("volume", 0)),
-                    "liquidity": float(event.get("liquidity", 0)),
-                    "image": details.get("image", event.get("image", "")),
-                    "time_since": calculate_time_since(event.get("createdAt", ""))
-                })
-
-                if len(markets) >= self.num_items:
-                    break
-
-            print(f"   {len(markets)} markets remained after filtering")
-            print(f"   Found {len(markets)} recently created non-sports/crypto markets")
-            return markets
-
-        except Exception as e:
-            print(f"   Warning: API error: {e}")
-            return []
-    
-    def fetch_breaking_news(self) -> List[Dict[str, Any]]:
-        """Fetch breaking news markets by scraping Polymarket breaking page"""
-        print("📰 Fetching breaking news markets...")
-        
-        try:
-            headers = {
-                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
-            }
-            
-            response = requests.get("https://polymarket.com/breaking", headers=headers, timeout=15, impersonate="chrome110")
-            response.raise_for_status()
-            
-            # Extract __NEXT_DATA__ JSON
-            next_data_match = re.search(r'<script id="__NEXT_DATA__"[^>]*>(.*?)</script>', response.text, re.DOTALL)
-            if not next_data_match:
-                return []
-                
-            next_data = json.loads(next_data_match.group(1))
-            
-            # Navigate to markets data
-            queries = next_data['props']['pageProps']['dehydratedState']['queries']
-            markets = []
-            
-            for query in queries:
-                query_key = query.get('queryKey', [])
-                # Check if 'biggest-movers' is in the query key (could be ['biggest-movers'] or ['biggest-movers', 'all'])
-                if isinstance(query_key, list) and 'biggest-movers' in query_key:
-                    raw_markets = query['state']['data']['markets']
-                    
-                    # Define filter keywords for breaking news
-                    sports_keywords = ['vs.', ' vs ', 'NFL', 'NBA', 'MLB', 'NHL', 'UFC', 'MMA']
-                    crypto_keywords = ['bitcoin', 'ethereum', 'btc', 'eth', 'solana', 'sol',
-                                      'above ___', 'below ___', 'price on', 'what price will']
-                    weather_keywords = ['temperature', 'degrees', 'weather']
-                    esports_keywords = ['LoL:', 'Dota', 'CS:GO', '(BO3)', '(BO5)']
-                    
-                    for market in raw_markets:
-                        title = market.get('question', 'Unknown')
-                        title_lower = title.lower()
-                        
-                        # Skip uninteresting markets
-                        if any(k in title or k in title_lower for k in sports_keywords):
-                            continue
-                        if any(k in title_lower for k in crypto_keywords):
-                            continue
-                        if any(k in title_lower for k in weather_keywords):
-                            continue
-                        if any(k in title or k in title_lower for k in esports_keywords):
-                            continue
-                        if 'up or down' in title_lower:
-                            continue
-                        
-                        # Get current price and calculate change
-                        current_price_raw = market.get('currentPrice', 0)
-                        if current_price_raw is None:
-                            current_price_raw = 0
-                        current_price = int(float(current_price_raw) * 100)
-
-                        price_change = market.get('livePriceChange', 0)
-                        if price_change is None:
-                            price_change = 0
-
-                        # Get the correct event slug (not market slug)
-                        event_slug = market.get('slug', '')  # fallback to market slug
-                        events = market.get('events', [])
-                        if events and len(events) > 0:
-                            # Use the event slug which gives the correct URL
-                            event_slug = events[0].get('slug', event_slug)
-
-                        # Get image, with fallback to fetch details if missing
-                        image_url = market.get('image', '')
-                        if not image_url and event_slug:
-                            details = self.fetch_market_details(event_slug)
-                            image_url = details.get('image', '')
-
-                        markets.append({
-                            "title": title,
-                            "slug": event_slug,
-                            "image": image_url,
-                            "current_price": current_price,
-                            "price_change": price_change,
-                            "volume": float(market.get('volume', 0))
-                        })
-                        
-                        if len(markets) >= self.num_items:
-                            break
-                    
-                    break
-            
-            return markets
-            
-        except Exception as e:
-            print(f"   Warning: Scraping error: {e}")
-            return []
-    
-    def fetch_hot_markets(self) -> List[Dict[str, Any]]:
-        """Fetch hot markets by 24hr volume from Gamma API"""
-        print("Fetching hot markets...")
-        
-        try:
-            response = requests.get(
-                "https://gamma-api.polymarket.com/events",
-                params={
-                    "order": "volume24hr",
-                    "ascending": "false",
-                    "active": "true",
-                    "limit": 100
-                },
-                timeout=10,
-                impersonate="chrome110"
-            )
-            response.raise_for_status()
-            
-            events = response.json()
-            markets = []
-            
-            # Define filter keywords for hot markets
-            sports_keywords = ['vs.', ' vs ', 'NFL', 'NBA', 'MLB', 'NHL', 'UFC', 'MMA']
-            esports_keywords = ['LoL:', 'Dota', 'CS:GO', '(BO3)', '(BO5)', 'Mobile Legends', 'MLBB']
-            crypto_short_keywords = ['above ___', 'below ___', 'price on', 'what price will',
-                                     '4pm et', '8pm et', '12pm et', '3:00pm', '3:15pm']
-            weather_keywords = ['temperature', 'degrees', 'weather', 'rain', 'snow']
-            
-            for event in events:
-                # Skip if closed or ended
-                if event.get('closed', False) or event.get('ended', False):
-                    continue
-                
-                title = event.get('title', '')
-                title_lower = title.lower()
-                slug = event.get('slug', '').lower()
-                
-                # Skip "Up or Down" markets
-                if 'up or down' in title_lower:
-                    continue
-                
-                # Skip short-term crypto markets
-                if any(keyword in title_lower or keyword in slug 
-                      for keyword in crypto_short_keywords):
-                    continue
-                
-                # Skip weather markets
-                if any(keyword in title_lower or keyword in slug 
-                      for keyword in weather_keywords):
-                    continue
-                
-                # Skip esports
-                if any(keyword in title or keyword in title_lower or keyword in slug 
-                      for keyword in esports_keywords):
-                    continue
-                
-                # Note: We're NOT filtering out ALL sports here, just closed/ended ones
-                # Active sports markets with high volume are legitimate hot markets
-                
-                # Fetch detailed data for image
-                details = self.fetch_market_details(event.get("slug", ""))
-                
-                markets.append({
-                    "title": event.get("title", "Unknown"),
-                    "slug": event.get("slug", ""),
-                    "volume24hr": float(event.get("volume24hr", 0)),
-                    "liquidity": float(event.get("liquidity", 0)),
-                    "image": details.get("image", "")
-                })
-                
-                if len(markets) >= self.num_items:
-                    break
-            
-            return markets
-            
-        except Exception as e:
-            print(f"   Warning: API error: {e}")
-            return []
-    
-    def fetch_sports_events(self) -> List[Dict[str, Any]]:
-        """Fetch upcoming sports events from Gamma API"""
-        print("Fetching sports events...")
-        
-        try:
-            # Get a larger set of events and filter for sports - paginated
-            # Fetch 200 total (2 pages of 100)
-            all_events = []
-            for offset in range(0, 200, 100):
-                response = requests.get(
-                    "https://gamma-api.polymarket.com/events",
-                    params={
-                        "active": "true",
-                        "closed": "false",
-                        "order": "volume24hr",  # Get popular events
-                        "ascending": "false",
-                        "limit": 100,
-                        "offset": offset
-                    },
-                    timeout=10,
-                    impersonate="chrome110"
-                )
-                if response.status_code == 200:
-                    page_data = response.json()
-                    all_events.extend(page_data)
-                    # If we got fewer than 100, we've reached the end
-                    if len(page_data) < 100:
-                        break
-                else:
-                    response.raise_for_status()
-                    break
-            
-            # Filter for sports-related titles - look for "vs" or "vs." which is common in sports
-            sports_keywords = ['vs.', ' vs ', 'NFL', 'NBA', 'MLB', 'NHL', 'UFC', 'MMA', 
-                             'Soccer', 'Football', 'Basketball', 'Baseball', 'Hockey', 
-                             'Tennis', 'Golf', 'Boxing', 'Premier League', 'Champions League', 
-                             'World Cup', 'Super Bowl', 'fight', 'bout', 'F1', 'Racing',
-                             'tournament', 'championship', 'Rookie of the Year', 'MVP']
-            
-            events = []
-            for event in all_events:
-                # Skip if the event is closed or ended
-                if event.get('closed', False) or event.get('ended', False):
-                    continue
-                    
-                title = event.get("title", "")
-                # Check if title contains sports keywords (case insensitive for some)
-                is_sport = False
-                for keyword in sports_keywords:
-                    if keyword in title or keyword.lower() in title.lower():
-                        is_sport = True
-                        break
-                
-                # Additional filters to exclude non-sports that might have "vs"
-                exclude_keywords = ['Trump', 'Biden', 'election', 'presidential', 'divorce', 
-                                  'coronavirus', 'COVID', 'vaccine', 'trading', 'price', 
-                                  'Bitcoin', 'Ethereum', 'stock', 'IPO', 'Supreme Court',
-                                  'LoL:', 'Dota', 'CS:GO', 'esports']  # Exclude esports
-                if is_sport and not any(ex in title for ex in exclude_keywords):
-                    events.append(event)
-                    
-                if len(events) >= self.num_items * 2:
-                    break
-            
-            markets = []
-            for event in events[:self.num_items]:
-                # Sports events already have image in the main response
-                image_url = event.get("image", "")
-                
-                # If no image in main data, try fetching details
-                if not image_url:
-                    details = self.fetch_market_details(event.get("slug", ""))
-                    image_url = details.get("image", "")
-                
-                # Calculate time to start or end
-                end_time = event.get("endDate", event.get("closedTime", ""))
-                time_to_start = calculate_time_remaining(end_time)
-                
-                # Determine sport type from title
-                title = event.get("title", "")
-                sport = "Sports"
-                if any(x in title for x in ["NFL", "football", "Packers", "Bears", "Cowboys", "Patriots"]):
-                    sport = "NFL"
-                elif any(x in title for x in ["NBA", "basketball", "Lakers", "Celtics", "Warriors"]):
-                    sport = "NBA"
-                elif any(x in title for x in ["MLB", "baseball", "Yankees", "Dodgers", "Astros", "Royals", "Phillies"]):
-                    sport = "MLB"
-                elif any(x in title for x in ["UFC", "MMA", "fight", "Boxing"]):
-                    sport = "UFC/MMA"
-                elif any(x in title for x in ["Soccer", "Premier League", "Champions League", "World Cup"]):
-                    sport = "Soccer"
-                elif "Golf" in title:
-                    sport = "Golf"
-                elif any(x in title for x in ["Tennis", "Wimbledon", "Open"]):
-                    sport = "Tennis"
-                elif "Super Bowl" in title:
-                    sport = "NFL"
-                
-                markets.append({
-                    "title": title,
-                    "slug": event.get("slug", ""),
-                    "startTime": end_time,
-                    "time_to_start": time_to_start,
-                    "volume": float(event.get("volume", 0)),
-                    "image": image_url,
-                    "sport": sport
-                })
-            
-            # If still no sports events found, return empty
-            if not markets:
-                print("   Warning: No sports events found")
-            
-            return markets
-            
-        except Exception as e:
-            print(f"   Warning: API error: {e}")
-            return []
-    
-    def fetch_whale_moves(self) -> List[Dict[str, Any]]:
-        """Fetch whale moves from Polymarket trades API"""
-        print("Fetching whale moves...")
-        
-        import time
-        
-        # Polymarket trades API endpoint
-        url = 'https://data-api.polymarket.com/trades'
-        params = {
-            'limit': 100,
-            'takerOnly': 'true',
-            'filterType': 'CASH',
-            'filterAmount': 50000
-        }
-        
-        max_retries = 3
-        timeout = 30
-        
-        for attempt in range(max_retries):
-            try:
-                if attempt > 0:
-                    wait_time = 2 ** attempt
-                    print(f"   Retry {attempt}/{max_retries} after {wait_time}s...")
-                    time.sleep(wait_time)
-                
-                response = requests.get(
-                    url,
-                    params=params,
-                    headers={'Content-Type': 'application/json'},
-                    timeout=timeout,
-                    impersonate="chrome110"
-                )
-                response.raise_for_status()
-                break
-                
-            except requests.exceptions.RequestException as e:
-                print(f"   Warning: API request attempt {attempt + 1}/{max_retries} failed: {e}")
-                if attempt == max_retries - 1:
-                    print(f"   Error: Failed after {max_retries} attempts")
-                    print("   Falling back to placeholder whale move data")
-                    return self._placeholder_whale_moves()
-        
-        # Process the successful response
-        try:
-            trades_data = response.json()
-        except json.JSONDecodeError as e:
-            print(f"   Error: Failed to parse API response: {e}")
-            print("   Falling back to placeholder whale move data")
-            return self._placeholder_whale_moves()
-        
-        if not isinstance(trades_data, list):
-            print(f"   Error: Expected array response, got {type(trades_data)}")
-            print("   Falling back to placeholder whale move data")
-            return self._placeholder_whale_moves()
-        
-        # Get current Unix timestamp
-        current_timestamp = int(time.time())
-        # 24 hours in seconds
-        twenty_four_hours_ago = current_timestamp - (24 * 60 * 60)
-        
-        # Sports slugs to filter out
-        sports_keywords = ['nba', 'nhl', 'epl', 'ucl', 'nfl']
-        
-        # Filter and process trades
-        whale_moves = []
-        
-        for trade in trades_data:
-            try:
-                # Filter out trades older than 24 hours
-                trade_timestamp = trade.get('timestamp', 0)
-                if trade_timestamp < twenty_four_hours_ago:
-                    continue
-                
-                # Filter out sports-related slugs
-                slug = trade.get('slug', '').lower()
-                if any(keyword in slug for keyword in sports_keywords):
-                    continue
-                
-                # Extract data
-                size = float(trade.get('size', 0))
-                price = float(trade.get('price', 0))
-                amount = size * price  # Calculate total amount
-                
-                title_text = trade.get('title', 'Unknown Market')
-                outcome = trade.get('outcome', '')
-                side = trade.get('side', '')  # BUY or SELL
-                event_slug = trade.get('eventSlug', '')
-                
-                # Build title with outcome: "OUTCOME" Market
-                if outcome:
-                    title = f'"{outcome}" {title_text}'
-                else:
-                    title = title_text
-                
-                # Use slug from trade, or generate from title
-                if event_slug:
-                    slug_for_display = event_slug
-                else:
-                    slug_for_display = title_text.lower().replace(' ', '-').replace('?', '').replace("'", '')[:50]
-                
-                # Get user information
-                user_name = trade.get('name', trade.get('pseudonym', 'Anonymous'))
-                
-                # Filter out trades from specific user
-                if user_name.lower() == 'kamakamakamata':
-                    continue
-                
-                profile_image = trade.get('profileImageOptimized') or trade.get('profileImage', '')
-                
-                # Get market icon if available
-                market_icon = trade.get('icon', '')
-                
-                whale_moves.append({
-                    "title": title,
-                    "market": title_text,
-                    "event": title_text,
-                    "slug": slug_for_display,
-                    "amount": amount,
-                    "outcome": outcome,  # Store for aggregation
-                    "side": side,  # Store for aggregation
-                    "user_name": user_name,
-                    "user_profile_url": user_name,
-                    "distinct_positions": 1,
-                    "timestamp": datetime.now().isoformat(),
-                    "profile_image": profile_image,  # Store for enrichment
-                    "market_icon": market_icon  # Store for enrichment
-                })
-                
-            except (KeyError, ValueError, TypeError) as e:
-                print(f"   Warning: Failed to process trade data: {e}")
-                continue
-        
-        # Aggregate trades: combine trades with same market, outcome, side, and user
-        print(f"   Aggregating {len(whale_moves)} trades...")
-        aggregated_moves = {}
-        
-        for move in whale_moves:
-            # Create aggregation key: (market, outcome, side, user_name)
-            agg_key = (
-                move.get('market', ''),
-                move.get('outcome', ''),
-                move.get('side', ''),
-                move.get('user_name', '')
-            )
-            
-            if agg_key in aggregated_moves:
-                # Combine with existing trade: sum amounts and update distinct_positions
-                existing = aggregated_moves[agg_key]
-                existing['amount'] += move['amount']
-                existing['distinct_positions'] += 1
-                # Update title: "OUTCOME" Market
-                outcome = move.get('outcome', '')
-                market = existing.get('market', '')
-                if outcome:
-                    existing['title'] = f'"{outcome}" {market}'
-                else:
-                    existing['title'] = market
-                # Keep the most recent profile_image and market_icon if current one is empty
-                if not existing.get('profile_image') and move.get('profile_image'):
-                    existing['profile_image'] = move.get('profile_image')
-                if not existing.get('market_icon') and move.get('market_icon'):
-                    existing['market_icon'] = move.get('market_icon')
-            else:
-                # First trade for this combination
-                aggregated_moves[agg_key] = move.copy()
-        
-        # Convert back to list
-        whale_moves = list(aggregated_moves.values())
-        print(f"   Aggregated to {len(whale_moves)} unique trades")
-        
-        # Sort by amount (descending) and limit to top 10
-        whale_moves.sort(key=lambda x: x['amount'], reverse=True)
-        whale_moves = whale_moves[:10]
-        
-        if not whale_moves:
-            print("   No whale moves found after filtering")
-            print("   Falling back to placeholder whale move data")
-            return self._placeholder_whale_moves()
-
-        print(f"   Found {len(whale_moves)} whale moves after filtering")
-        
-        # Enrich with market images and user images
-        whale_moves = self._enrich_whale_moves_with_images(whale_moves)
-
-        return whale_moves[:self.num_items]
-
-    def _enrich_whale_moves_with_images(self, whale_moves: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
-        """Enrich whale moves with market images and user profile images."""
-        print("   Fetching market images for whale moves...")
-        default_user_image = "https://res.cloudinary.com/db1zelfhi/image/upload/v1765869030/polygraph/images/jhvsxfndu0boigqz3kjw.png"
-        
-        for move in whale_moves:
-            # Use profile image from API if available, otherwise use default
-            profile_image = move.pop('profile_image', '')  # Remove from dict after getting
-            if profile_image and profile_image.strip():
-                move['user_image'] = profile_image
-            else:
-                move['user_image'] = default_user_image
-            
-            # Use market icon from API if available, otherwise fetch using slug
-            market_icon = move.pop('market_icon', '')  # Remove from dict after getting
-            if market_icon and market_icon.strip():
-                move['market_image'] = market_icon
-                print(f"      ✓ Using icon from API for {move.get('market', 'Unknown')[:50]}")
-            else:
-                # Fetch market image using slug as fallback
-                slug = move.get('slug', '')
-                if slug:
-                    try:
-                        details = self.fetch_market_details(slug)
-                        market_image = details.get('image', '')
-                        move['market_image'] = market_image
-                        if market_image:
-                            print(f"      ✓ Found image for {slug[:50]}")
-                        else:
-                            print(f"      ⚠ No image found for {slug[:50]}")
-                    except Exception as e:
-                        print(f"      ⚠ Error fetching image for {slug[:50]}: {e}")
-                        move['market_image'] = ''
-                else:
-                    print(f"      ⚠ No slug available for market: {move.get('market', 'Unknown')}")
-                    move['market_image'] = ''
-        
-        return whale_moves
-
-    def _placeholder_whale_moves(self) -> List[Dict[str, Any]]:
-        """Return fallback whale move data when Polymarket API is unavailable."""
-        placeholder_moves = _fresh_placeholder_whales()
-        print(f"   Using {len(placeholder_moves)} placeholder whale entries")
-        # Enrich placeholder moves with images too
-        placeholder_moves = self._enrich_whale_moves_with_images(placeholder_moves)
-        return placeholder_moves[:self.num_items]
-
-    def _placeholder_comments(self) -> List[Dict[str, Any]]:
-        """Return fallback comment data when Polymarket APIs are unavailable."""
-        comments = _fresh_placeholder_comments()
-        print(f"   Using {len(comments)} placeholder comment entries")
-        return comments[:self.num_items]
-    
-    def fetch_top_comments(self) -> List[Dict[str, Any]]:
-        """Fetch top comments with smart filtering to avoid staleness"""
-        print("Fetching top comments with smart filtering...")
-
-        try:
-            from dateutil import parser
-            from datetime import timezone
-            now = datetime.now(timezone.utc)
-            
-            # Get top active markets for variety
-            events_response = requests.get(
-                "https://gamma-api.polymarket.com/events",
-                params={
-                    "order": "volume24hr",
-                    "ascending": "false",
-                    "active": "true",
-                    "limit": 30  # Check more markets for variety
-                },
-                timeout=10,
-                impersonate="chrome110"
-            )
-            events_response.raise_for_status()
-            events = events_response.json()
-
-            all_comments = []
-            markets_with_comments = set()  # Track which markets we've gotten comments from
-            
-            # Strategy 1: Get recent high-engagement comments (last 7 days)
-            for event in events[:25]:
-                event_id = event.get('id')
-                event_title = event.get('title', 'Unknown Market')
-                event_slug = event.get('slug', '')
-                
-                if not event_id:
-                    continue
-                
-                # Limit comments per market for variety
-                if event_title in markets_with_comments and len(markets_with_comments) < 10:
-                    continue
-
-                try:
-                    # Fetch both recent and top comments
-                    for order_type in ["reactionCount", "createdAt"]:
-                        comments_response = requests.get(
-                            "https://gamma-api.polymarket.com/comments",
-                            params={
-                                "order": order_type,
-                                "ascending": "false",
-                                "limit": 10,
-                                "parent_entity_type": "Event",
-                                "parent_entity_id": event_id
-                            },
-                            timeout=5,
-                            impersonate="chrome110"
-                        )
-
-                        if comments_response.status_code == 200:
-                            comments = comments_response.json()
-                            
-                            for comment in comments:
-                                body = comment.get('body', '').strip()
-                                
-                                # Filter out short/low-quality comments
-                                if len(body) < 20:  # Too short
-                                    continue
-                                if body.lower() in ['nice', 'lol', 'based', 'bullish', 'bearish', 'lfg', 'hodl', 'moon']:
-                                    continue
-                                    
-                                created_at = comment.get('createdAt', '')
-                                likes = comment.get('reactionCount', 0)
-                                
-                                # Calculate age in days
-                                age_days = 1000  # Default to very old
-                                if created_at:
-                                    try:
-                                        comment_date = parser.parse(created_at)
-                                        if comment_date.tzinfo is None:
-                                            comment_date = comment_date.replace(tzinfo=timezone.utc)
-                                        age = now - comment_date
-                                        age_days = max(age.total_seconds() / 86400, 0.1)  # Minimum 0.1 days
-                                    except:
-                                        continue
-                                
-                                # Skip if too old (>14 days) unless it has exceptional engagement
-                                if age_days > 14 and likes < 20:
-                                    continue
-                                
-                                # Skip if too recent with no engagement
-                                if age_days < 2 and likes < 3:
-                                    continue
-                                    
-                                # Skip comments with 0 likes unless they're brand new and substantial
-                                if likes == 0 and (age_days > 1 or len(body) < 100):
-                                    continue
-                                
-                                # Calculate weighted score: likes / (age_in_days + 1)
-                                # This favors recent comments with good engagement
-                                weighted_score = likes / (age_days + 1)
-                                
-                                # Boost very recent highly engaging comments
-                                if age_days < 2 and likes > 10:
-                                    weighted_score *= 2
-                                
-                                all_comments.append({
-                                    "comment": body,
-                                    "author": comment.get('profile', {}).get('name', 
-                                             comment.get('profile', {}).get('pseudonym', 'Anonymous')),
-                                    "market_title": event_title,
-                                    "slug": event_slug,
-                                    "likes": likes,
-                                    "createdAt": created_at,
-                                    "age_days": age_days,
-                                    "weighted_score": weighted_score
-                                })
-                                
-                                markets_with_comments.add(event_title)
-                                
-                except Exception as e:
-                    continue
-
-            # Remove duplicates (same comment text)
-            seen_comments = set()
-            unique_comments = []
-            for comment in all_comments:
-                comment_key = comment['comment'][:100].lower()  # Use first 100 chars as key
-                if comment_key not in seen_comments:
-                    seen_comments.add(comment_key)
-                    unique_comments.append(comment)
-            
-            # Sort by weighted score
-            unique_comments.sort(key=lambda x: x['weighted_score'], reverse=True)
-            
-            # Take top N, ensuring variety
-            final_comments = []
-            markets_included = set()
-            
-            for comment in unique_comments:
-                # Limit 2 comments per market for variety
-                market_count = sum(1 for c in final_comments if c['market_title'] == comment['market_title'])
-                if market_count >= 2:
-                    continue
-                    
-                final_comments.append(comment)
-                markets_included.add(comment['market_title'])
-                
-                if len(final_comments) >= self.num_items:
-                    break
-            
-            # Log some stats for debugging
-            if final_comments:
-                avg_age = sum(c['age_days'] for c in final_comments) / len(final_comments)
-                print(f"   Selected {len(final_comments)} comments from {len(markets_included)} markets")
-                print(f"   Average age: {avg_age:.1f} days (vs 36 days before)")
-                return final_comments
-
-            print("   No qualifying comments returned - using placeholder comments")
-            return self._placeholder_comments()
-
-        except Exception as e:
-            print(f"   Warning: API error: {e}")
-            return self._placeholder_comments()
-
-    def display_menu(self) -> None:
-        """Display the menu of available markets"""
-        self.build_menu(verbose=True)
-
-    def build_menu(self, verbose: bool = False) -> List[Dict[str, Any]]:
-        """Populate structured menu entries and optionally print them."""
-        sections = [
-            ("ending_soon", "Ending Soon", self.fetch_ending_soon, "??"),
-            ("just_listed", "New", self.fetch_just_listed, "??"),
-            ("breaking_news", "Breaking News", self.fetch_breaking_news, "??"),
-            ("hot_markets", "Hot Markets", self.fetch_hot_markets, "??"),
-            ("whale_moves", "Whale Moves", self.fetch_whale_moves, "??"),
-            ("top_comments", "Top Comments", self.fetch_top_comments, "??"),
-            ("sports_events", "Top Sports", self.fetch_sports_events, "??"),
-        ]
-
-        if verbose:
-            print("\n" + "=" * 60)
-            print("POLYMARKET DATA EMAIL MENU")
-            print("=" * 60)
-
-        menu_entries: List[Dict[str, Any]] = []
-        counter = 1
-        self.menu_items = {}
-
-        for key, label, fetcher, emoji in sections:
-            items = fetcher()
-
-            if verbose:
-                print(f"\n{emoji} {label.upper()}")
-                print("-" * 40)
-
-            for item in items:
-                self.menu_items[counter] = (key, item)
-                url = self._resolve_item_url(key, item)
-                # Create Dub tracking link if API key is set and not skipping
-                if url and not self.skip_dub:
-                    title = self._format_menu_entry_text(key, item, url)[0].strip('"')
-                    url = create_dub_tracking_link(url, title, tags=["polygraph"], skip_dub=self.skip_dub)
-                menu_entries.append({
-                    "id": counter,
-                    "category": key,
-                    "label": label,
-                    "emoji": emoji,
-                    "data": item,
-                    "url": url,
-                })
-
-                if verbose:
-                    title_line, detail_line = self._format_menu_entry_text(key, item, url)
-                    print(f"{counter}. {title_line}")
-                    if detail_line:
-                        print(detail_line)
-
-                counter += 1
-
-        if verbose:
-            print("\n" + "=" * 60)
-
-        return menu_entries
-
-    def _format_menu_entry_text(self, category: str, data: Dict[str, Any], url: str) -> Tuple[str, str]:
-        """Return printable title/detail strings for menu entries."""
-        if category == "top_comments":
-            text = data.get("comment", "").strip()
-            snippet = text if len(text) <= 200 else f"{text[:200]}..."
-            likes = data.get("likes", 0)
-            market = data.get("market_title", "Unknown Market")
-            author = data.get("author", "Anonymous")
-            detail = f"   - {author} on {market} (?? {likes})"
-            return f"\"{snippet}\"", detail
-
-        if category == "whale_moves":
-            title = data.get("title", "Large position")
-            market = data.get("market", "N/A")
-            return title, f"   Market: {market}"
-
-        title = data.get("title") or data.get("event") or "Untitled market"
-        detail = f"   {url}" if url else ""
-        return title, detail
-
-    def _resolve_item_url(self, category: str, data: Dict[str, Any]) -> str:
-        """Resolve the canonical Polymarket link for a menu entry."""
-        slug = data.get("slug") or data.get("market_slug")
-        if slug:
-            return f"https://polymarket.com/event/{slug}"
-
-        if category == "whale_moves" and data.get("event"):
-            event_slug = re.sub(r"[^a-z0-9-]", "-", data["event"].lower()).strip("-")
-            if event_slug:
-                return f"https://polymarket.com/event/{event_slug}"
-
-        return data.get("link", "")
-    def get_user_selections(self) -> List[tuple]:
-        """Get user's market selections with optional 's' suffix for sports section"""
-        print("\n>>> Enter Your Selections (comma-separated numbers):")
-        print("    Example: 2, 3, 4, 9, 10, 35")
-        print("    Add 's' to move to Sports section: 21s, 25s")
-        print("    Or press Enter to skip selection")
-        print("    Or input 'z' to go back\n")
-
-        user_input = input(">>> ").strip()
-
-        if user_input.lower() == 'z':
-            return 'BACK'
-
-        if not user_input:
-            return []
-
-        try:
-            valid_selections = []
-            for item in user_input.split(','):
-                item = item.strip()
-
-                # Check if item has 's' suffix for sports override
-                force_sports = False
-                if item.endswith('s') or item.endswith('S'):
-                    force_sports = True
-                    item = item[:-1]  # Remove the 's' suffix
-
-                # Convert to integer
-                sel = int(item)
-
-                # Validate selection
-                if sel in self.menu_items:
-                    valid_selections.append((sel, force_sports))
-                else:
-                    print(f"Warning: Selection {sel} is invalid (out of range)")
-
-            return valid_selections
-        except ValueError as e:
-            print(f"Error: Invalid input. Please enter comma-separated numbers (optionally with 's' suffix).")
-            return self.get_user_selections()
-    
-    def get_poly_archive_tweets(self) -> List[Dict[str, Any]]:
-        """Prompt user for poly_archive tweet image URLs and associated markets"""
-        print("\n" + "="*60)
-        print("📸 POLY_ARCHIVE TWEET SCREENSHOTS (optional)")
-        print("="*60)
-        print("Add tweet screenshots from @poly_archive to Top Comments section")
-        print("Right-click on tweet image → 'Copy Image Address' → paste here")
-        print("Press Enter to skip, or input 'z' to go back\n")
-
-        tweets = []
-
-        while True:
-            image_url = input(">>> Image URL from poly_archive tweet (or Enter to finish, 'z' to go back): ").strip()
-
-            if image_url.lower() == 'z':
-                return 'BACK'
-
-            if not image_url:
-                break
-
-            # Validate it's an image URL
-            if not any(ext in image_url.lower() for ext in ['.jpg', '.jpeg', '.png', '.gif', 'pbs.twimg.com']):
-                print("   ⚠️  Doesn't look like an image URL. Please paste the direct image URL")
-                continue
-
-            # Prompt for the market slug
-            print(">>> Polymarket market slug or full URL for this tweet (or 'z' to go back):")
-            market_input = input(">>> ").strip()
-
-            if market_input.lower() == 'z':
-                return 'BACK'
-
-            if not market_input:
-                print("   ⚠️  Market required. Skipping this tweet.")
-                continue
-
-            # Extract slug from URL if full URL provided
-            if 'polymarket.com' in market_input:
-                # Extract slug from URL like https://polymarket.com/event/slug-here?tid=123
-                import re
-                slug_match = re.search(r'/event/([^?/#]+)', market_input)
-                if slug_match:
-                    market_slug = slug_match.group(1)
-                else:
-                    print("   ⚠️  Could not extract market slug from URL. Skipping.")
-                    continue
-            else:
-                market_slug = market_input
-
-            # Download the image
-            print(f"   Downloading image...")
-            try:
-                headers = {
-                    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
-                }
-                img_response = requests.get(image_url, headers=headers, timeout=10, impersonate="chrome110")
-
-                if img_response.status_code != 200:
-                    print(f"   ⚠️  Failed to download image: HTTP {img_response.status_code}")
-                    continue
-
-                image_data = img_response.content
-
-            except Exception as e:
-                print(f"   ⚠️  Error downloading image: {e}")
-                continue
-
-            # Upload to Cloudinary (no resizing)
-            date_folder = datetime.now().strftime("%-m-%-d-%y")
-            filename = f"poly_archive_{len(tweets)}.png"
-
-            print(f"   Uploading to Cloudinary...")
-            cloudinary_url = upload_to_cloudinary(image_data, filename, date_folder, apply_transform=False)
-
-            if cloudinary_url and cloudinary_url != "https://via.placeholder.com/48":
-                tweets.append({
-                    "market_slug": market_slug,
-                    "image_url": cloudinary_url,
-                    "is_tweet_screenshot": True
-                })
-                print(f"   ✅ Added tweet screenshot for market: {market_slug}\n")
-            else:
-                print(f"   ⚠️  Failed to upload image. Skipping.\n")
-
-        if tweets:
-            print(f"\n✅ Added {len(tweets)} poly_archive tweet screenshot(s)\n")
-
-        return tweets
-
-    def organize_selections(self, selections: List[tuple]) -> Dict[str, List[Dict]]:
-        """Organize selected markets by category with sports override support"""
-        organized = defaultdict(list)
-
-        for item in selections:
-            # Handle both old format (int) and new format (tuple)
-            if isinstance(item, tuple):
-                sel, force_sports = item
-            else:
-                sel = item
-                force_sports = False
-
-            category, market_data = self.menu_items[sel]
-
-            # Override category to sports_events if 's' suffix was used
-            if force_sports and category != "sports_events":
-                print(f"   Moving item {sel} from {category} to sports section")
-                category = "sports_events"
-
-            organized[category].append(market_data)
-
-        return organized
-
-    def collect_top_stories(self) -> List[Dict[str, str]]:
-        """Collect top 3 stories for the header section"""
-        print("\n" + "="*60)
-        print("📰 TOP STORIES & INTRO")
-        print("="*60)
-
-        # First collect intro text
-        print("\n📝 INTRO TEXT (appears below date, above Top Stories):")
-        print("   This is your opening message or summary")
-        print("   Press Enter to skip, or input 'z' to go back\n")
-        intro_text = input("Intro text: ").strip()
-
-        if intro_text.lower() == 'z':
-            return 'BACK'
-
-        print("\nNow enter 3 top stories to highlight")
-        print("Press Enter to skip any story, or input 'z' to go back\n")
-
-        stories = []
-
-        for i in range(1, 4):
-            print(f"STORY {i}:")
-            print("-" * 30)
-
-            headline = input(f"   Headline {i}: ").strip()
-            if headline.lower() == 'z':
-                return 'BACK'
-            if not headline:
-                print(f"   Skipping story {i}")
-                continue
-
-            body_text = input(f"   Copy/Description: ").strip()
-            if body_text.lower() == 'z':
-                return 'BACK'
-            if not body_text:
-                body_text = ""
-
-            # Get the Polymarket link
-            link = input(f"   Polymarket URL (or slug): ").strip()
-            if link.lower() == 'z':
-                return 'BACK'
-            if link:
-                # If it's just a slug, convert to full URL
-                if not link.startswith("http"):
-                    link = f"https://polymarket.com/event/{link}"
-
-                # Create tracking link if Dub is enabled
-                tracking_link = create_dub_tracking_link(link, headline, tags=["polygraph"], skip_dub=self.skip_dub)
-
-                stories.append({
-                    "headline": headline,
-                    "body": body_text,
-                    "link": tracking_link
-                })
-                print(f"   ✅ Added story {i}")
-            else:
-                print(f"   ⚠️  Skipping story {i} - no link provided")
-
-        result = {"stories": stories}
-        if intro_text:
-            result["intro_text"] = intro_text
-
-        if stories:
-            print(f"\n✅ Added {len(stories)} top stories")
-        if intro_text:
-            print(f"✅ Added intro text")
-        if not stories and not intro_text:
-            print("\nℹ️  No top stories or intro added")
-
-        return result
-
-    def run(self):
-        """Main execution flow with support for going back"""
-        print("\nStarting Polymarket Data Collection")
-        print(f"   Items per section: {self.num_items}")
-
-        # Check for Cloudinary
-        if os.getenv('CLOUDINARY_URL'):
-            print("✅ Cloudinary configured - tweet images will be uploaded")
-        else:
-            print("⚠️  CLOUDINARY_URL not set - tweet images won't be uploaded")
-
-        # Check for Dub API
-        if os.getenv('DUB_API_KEY'):
-            print("✅ Dub.co configured - links will be tracked")
-        else:
-            print("⚠️  DUB_API_KEY not set - links won't be tracked")
-
-        # State management for going back
-        step = 1
-        selections = None
-        organized = None
-        poly_archive_tweets = None
-        top_stories = None
-
-        while True:
-            if step == 1:
-                # Display menu and get selections
-                self.display_menu()
-                selections = self.get_user_selections()
-
-                if selections == 'BACK':
-                    print("\n⚠️  Can't go back - this is the first step")
-                    continue
-
-                if not selections:
-                    print("\nWarning: No selections made. Exiting.")
-                    return
-
-                print(f"\nSelected items: {[s[0] if isinstance(s, tuple) else s for s in selections]}")
-                organized = self.organize_selections(selections)
-                step = 2
-
-            elif step == 2:
-                # Get poly_archive tweets
-                poly_archive_tweets = self.get_poly_archive_tweets()
-
-                if poly_archive_tweets == 'BACK':
-                    print("\n↩️  Going back to market selections...\n")
-                    step = 1
-                    continue
-
-                # Add poly_archive tweets to top_comments section
-                if poly_archive_tweets:
-                    if "top_comments" not in organized:
-                        organized["top_comments"] = []
-                    organized["top_comments"].extend(poly_archive_tweets)
-
-                step = 3
-
-            elif step == 3:
-                # Collect top stories
-                top_stories = self.collect_top_stories()
-
-                if top_stories == 'BACK':
-                    print("\n↩️  Going back to poly_archive tweets...\n")
-                    # Remove previously added tweets from organized
-                    if poly_archive_tweets and "top_comments" in organized:
-                        for tweet in poly_archive_tweets:
-                            if tweet in organized["top_comments"]:
-                                organized["top_comments"].remove(tweet)
-                    step = 2
-                    continue
-
-                # All steps complete
-                break
-
-        # Summary
-        print("\n✅ Data collection complete!")
-        print(f"   Total sections: {len(organized)}")
-        print(f"   Total items: {len(selections)}")
-
-        for category, items in organized.items():
-            print(f"   • {category}: {len(items)} items")
-
-        print(f"\n🎉 Done! Data is ready for use.")
-
-
-def main():
-    """Main entry point"""
-    parser = argparse.ArgumentParser(description='Collect Polymarket Data (Markets, Comments, Whale Moves)')
-    parser.add_argument(
-        'num_items',
-        type=int,
-        nargs='?',
-        default=10,
-        help='Number of items per section (default: 10)'
-    )
-    parser.add_argument(
-        '--selections',
-        type=str,
-        help='Comma-separated list of selections (e.g., "1,3,5,7,10")'
-    )
-    parser.add_argument(
-        '--nodub',
-        action='store_true',
-        help='Skip creating Dub tracking links, use direct Polymarket URLs'
-    )
-    
-    args = parser.parse_args()
-    
-    # Validate num_items
-    if args.num_items < 1:
-        print("Error: Number of items must be at least 1")
-        sys.exit(1)
-    
-    if args.num_items > 50:
-        print("Warning: Large number of items may make the menu hard to navigate")
-    
-    # Create and run generator with nodub flag
-    generator = PolymarketEmailGenerator(num_items=args.num_items, skip_dub=args.nodub)
-    
-    # Show status if nodub is enabled
-    if args.nodub:
-        print("ℹ️  Dub tracking links disabled - using direct Polymarket URLs")
-    
-    # If selections provided via command line, use them
-    if args.selections:
-        generator.display_menu()
-
-        # Parse selections with 's' suffix support
-        valid_selections = []
-        for item in args.selections.split(','):
-            item = item.strip()
-
-            # Check for 's' suffix
-            force_sports = False
-            if item.endswith('s') or item.endswith('S'):
-                force_sports = True
-                item = item[:-1]
-
-            try:
-                sel = int(item)
-                if sel in generator.menu_items:
-                    valid_selections.append((sel, force_sports))
-                else:
-                    print(f"Warning: Selection {sel} is invalid (out of range)")
-            except ValueError:
-                print(f"Warning: Invalid selection '{item}'")
-
-        print(f"\nUsing command-line selections: {[s[0] if isinstance(s, tuple) else s for s in valid_selections]}")
-
-        if valid_selections:
-            # Organize selections
-            organized = generator.organize_selections(valid_selections)
-
-            # Collect top stories
-            top_stories = generator.collect_top_stories()
-
-            # Summary
-            print("\n✅ Data collection complete!")
-            print(f"   Total sections: {len(organized)}")
-            print(f"   Total items: {len(valid_selections)}")
-            for category, items in organized.items():
-                print(f"   • {category}: {len(items)} items")
-            print(f"\n🎉 Done! Data is ready for use.")
-    else:
-        generator.run()
-
-
-if __name__ == "__main__":
-    main()
-

app/fetch_menu_data.py

@@ -1,29 +1,30 @@
 #!/usr/bin/env python3
 """
-Standalone script to fetch Polymarket menu data and export as JSON.
-Run this on your local machine to avoid network timeout issues.
+Standalone script to fetch Kalshi menu data and export as JSON.
+Run this on your local machine to fetch data from Kalshi's public API.
 
 Usage:
-    python fetch_menu_data.py [--num-items N] [--output output.json]
+    python fetch_kalshi_menu.py [--num-items N] [--output output.json]
+
+Examples:
+    python fetch_kalshi_menu.py
+    python fetch_kalshi_menu.py --num-items 15 --output kalshi_data.json
+    python fetch_kalshi_menu.py --verbose
 """
 
 import argparse
 import json
 import sys
-from datetime import datetime
+from datetime import datetime, timezone
 from pathlib import Path
-from dotenv import load_dotenv
-
-# Load environment variables from .env file
-load_dotenv()
 
-# Import the generator from email3.py
-from email3 import PolymarketEmailGenerator
+# Import the generator from kalshi_data.py
+from kalshi_data import KalshiDataGenerator
 
 
 def main():
     parser = argparse.ArgumentParser(
-        description="Fetch Polymarket menu data and export as JSON"
+        description="Fetch Kalshi menu data and export as JSON"
     )
     parser.add_argument(
         "--num-items",
@@ -31,61 +32,62 @@ def main():
         default=10,
         help="Number of items per section (default: 10)",
     )
-    parser.add_argument(
-        "--skip-dub",
-        action="store_true",
-        help="Skip creating Dub tracking links, use direct Polymarket URLs",
-    )
     parser.add_argument(
         "--output",
         type=str,
-        default="menu_data.json",
-        help="Output JSON file path (default: menu_data.json)",
+        default="kalshi_menu_data.json",
+        help="Output JSON file path (default: kalshi_menu_data.json)",
     )
-    
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Print detailed menu output during fetch",
+    )
+
     args = parser.parse_args()
-    
-    print("🚀 Starting Polymarket menu data fetch...")
+
+    print("🚀 Starting Kalshi menu data fetch...")
     print(f"   Items per section: {args.num_items}")
-    print(f"   Skip Dub links: {args.skip_dub}")
+    print(f"   Output file: {args.output}")
     print()
-    
+
     try:
         # Create generator instance
-        generator = PolymarketEmailGenerator(
-            num_items=args.num_items,
-            skip_dub=args.skip_dub,
-        )
-        
-        # Build menu (this fetches all data including breaking news)
-        print("📊 Fetching menu data...")
-        menu_entries = generator.build_menu(verbose=True)
-        
+        generator = KalshiDataGenerator(num_items=args.num_items)
+
+        # Build menu (this fetches all data)
+        print("📊 Fetching menu data from Kalshi API...")
+        menu_entries = generator.build_menu(verbose=args.verbose)
+
         # Format as MenuResponse structure
         menu_response = {
-            "generated_at": datetime.utcnow().isoformat(),
+            "generated_at": datetime.now(timezone.utc).isoformat(),
+            "source": "Kalshi API",
+            "api_base": "https://api.elections.kalshi.com/trade-api/v2",
             "items": menu_entries,
         }
-        
+
         # Write to JSON file
         output_path = Path(args.output)
         with open(output_path, "w", encoding="utf-8") as f:
             json.dump(menu_response, f, indent=2, ensure_ascii=False)
-        
+
         print()
         print(f"✅ Success! Menu data exported to: {output_path}")
         print(f"   Total items: {len(menu_entries)}")
-        
+
         # Show breakdown by category
         categories = {}
         for item in menu_entries:
             cat = item.get("category", "unknown")
             categories[cat] = categories.get(cat, 0) + 1
-        
-        print(f"   Categories: {categories}")
-        
+
+        print("   Breakdown by section:")
+        for cat, count in categories.items():
+            print(f"      • {cat}: {count} items")
+
         return 0
-        
+
     except KeyboardInterrupt:
         print("\n⚠️  Interrupted by user")
         return 1
@@ -97,5 +99,4 @@ def main():
 
 
 if __name__ == "__main__":
-    sys.exit(main())
-
+    sys.exit(main())

app/kalshi_data.py

@@ -0,0 +1,819 @@
+#!/usr/bin/env python3
+"""
+Kalshi Data Collection Script
+Fetches and organizes market data from Kalshi's public API
+
+Sections implemented:
+- Ending Soon: Markets closing within the next few days
+- Just Listed: Recently created markets
+- Biggest Movers: Markets with largest 24h price changes (replaces "Breaking News")
+- Hot Markets: Markets with highest 24h volume
+- Sports Events: Sports-related markets
+
+Note: Whale Moves is NOT implemented as Kalshi's API does not expose trader identity.
+"""
+
+import argparse
+import json
+import sys
+from datetime import datetime, timezone
+from typing import Dict, List, Any, Optional
+import requests
+from collections import defaultdict
+
+# Kalshi API base URL (works for all markets, not just elections)
+BASE_URL = "https://api.elections.kalshi.com/trade-api/v2"
+
+
+def calculate_time_remaining(close_time_str: str) -> str:
+    """Calculate time remaining until market closes"""
+    try:
+        close_time = datetime.fromisoformat(close_time_str.replace('Z', '+00:00'))
+        now = datetime.now(timezone.utc)
+        delta = close_time - now
+
+        if delta.total_seconds() <= 0:
+            return "Closed"
+        elif delta.days > 0:
+            return f"{delta.days}d"
+        elif delta.seconds > 3600:
+            hours = delta.seconds // 3600
+            return f"{hours}h"
+        else:
+            minutes = delta.seconds // 60
+            return f"{minutes}m"
+    except Exception:
+        return "Soon"
+
+
+def calculate_time_since(created_time_str: str) -> str:
+    """Calculate time since market was created"""
+    try:
+        if not created_time_str:
+            return "Recently"
+
+        created_time = datetime.fromisoformat(created_time_str.replace('Z', '+00:00'))
+        now = datetime.now(timezone.utc)
+        delta = now - created_time
+        total_seconds = delta.total_seconds()
+
+        if delta.days > 30:
+            months = delta.days // 30
+            return f"{months}mo ago"
+        elif delta.days > 0:
+            return f"{delta.days}d ago"
+        elif total_seconds > 3600:
+            hours = int(total_seconds // 3600)
+            return f"{hours}h ago"
+        elif total_seconds > 60:
+            minutes = int(total_seconds // 60)
+            return f"{minutes}m ago"
+        else:
+            return "Just now"
+    except Exception:
+        return "Recently"
+
+
+class KalshiDataGenerator:
+    """Main class for collecting and organizing Kalshi market data"""
+
+    # Keywords to exclude for non-sports/non-weather sections
+    WEATHER_KEYWORDS = [
+        'temperature', 'degrees', 'rain', 'snow', 'weather', 'storm',
+        'hurricane', 'tornado', 'hotter', 'colder', 'warmest', 'coldest',
+        'precipitation', 'humidity', 'forecast', 'climate', 'kxhighny',
+        'kxlowny', 'kxhighla', 'kxhighchi', 'temp'
+    ]
+
+    SPORTS_KEYWORDS = [
+        'vs.', ' vs ', 'NFL', 'NBA', 'MLB', 'NHL', 'UFC', 'MMA',
+        'Soccer', 'Football', 'Basketball', 'Baseball', 'Hockey',
+        'Tennis', 'Golf', 'Boxing', 'Premier League', 'Champions League',
+        'World Cup', 'Super Bowl', 'fight', 'bout', 'F1', 'Racing',
+        'tournament', 'championship', 'Rookie of the Year', 'MVP',
+        'Heisman', 'playoffs', 'ncaa', 'bowl game'
+    ]
+
+    CRYPTO_KEYWORDS = [
+        'bitcoin', 'ethereum', 'btc', 'eth', 'solana', 'sol', 'xrp',
+        'crypto', 'doge', 'cardano', 'bnb', 'polygon', 'avalanche'
+    ]
+
+    def __init__(self, num_items: int = 10):
+        """Initialize with number of items per section"""
+        self.num_items = num_items
+        self.menu_items = {}
+        self.event_metadata_cache = {}
+
+    def _api_get(self, endpoint: str, params: Optional[Dict] = None) -> Dict:
+        """Make a GET request to the Kalshi API"""
+        url = f"{BASE_URL}{endpoint}"
+        try:
+            response = requests.get(url, params=params, timeout=15)
+            response.raise_for_status()
+            return response.json()
+        except requests.exceptions.RequestException as e:
+            print(f"   Warning: API request failed: {e}")
+            return {}
+
+    def _fetch_all_markets(self, status: str = "open", limit: int = 1000) -> List[Dict]:
+        """Fetch all markets with pagination support"""
+        all_markets = []
+        cursor = None
+
+        while True:
+            params = {"status": status, "limit": min(limit, 1000), "mve_filter": "exclude"}
+            if cursor:
+                params["cursor"] = cursor
+
+            data = self._api_get("/markets", params)
+            markets = data.get("markets", [])
+            all_markets.extend(markets)
+
+            cursor = data.get("cursor", "")
+            if not cursor or len(markets) < params["limit"]:
+                break
+
+        return all_markets
+
+    def _fetch_event_metadata(self, event_ticker: str) -> Dict:
+        """Fetch event metadata including images"""
+        if event_ticker in self.event_metadata_cache:
+            return self.event_metadata_cache[event_ticker]
+
+        try:
+            data = self._api_get(f"/events/{event_ticker}/metadata")
+            self.event_metadata_cache[event_ticker] = data
+            return data
+        except Exception:
+            return {}
+
+    def _enrich_with_images(self, markets: List[Dict]) -> List[Dict]:
+        """Fetch and add images for a list of markets (call only on final items)"""
+        if not markets:
+            return markets
+        
+        print(f"   Fetching images for {len(markets)} markets...")
+        for market in markets:
+            event_ticker = market.get("event_ticker", "")
+            if event_ticker:
+                metadata = self._fetch_event_metadata(event_ticker)
+                market["image"] = metadata.get("image_url", "")
+            else:
+                market["image"] = ""
+        return markets
+
+    def _is_weather_market(self, title: str, ticker: str) -> bool:
+        """Check if market is weather-related"""
+        title_lower = title.lower()
+        ticker_lower = ticker.lower()
+        return any(kw.lower() in title_lower or kw.lower() in ticker_lower
+                   for kw in self.WEATHER_KEYWORDS)
+
+    def _is_sports_market(self, title: str, ticker: str, category: str = "") -> bool:
+        """Check if market is sports-related"""
+        title_lower = title.lower()
+        ticker_lower = ticker.lower()
+        category_lower = category.lower() if category else ""
+
+        if category_lower == "sports":
+            return True
+
+        return any(kw.lower() in title_lower or kw.lower() in ticker_lower
+                   for kw in self.SPORTS_KEYWORDS)
+
+    def _is_crypto_market(self, title: str, ticker: str) -> bool:
+        """Check if market is crypto-related"""
+        title_lower = title.lower()
+        ticker_lower = ticker.lower()
+        return any(kw.lower() in title_lower or kw.lower() in ticker_lower
+                   for kw in self.CRYPTO_KEYWORDS)
+
+    def fetch_ending_soon(self) -> List[Dict[str, Any]]:
+        """Fetch markets ending soon - excludes sports, weather, crypto"""
+        print("Fetching markets ending soon...")
+
+        now = datetime.now(timezone.utc)
+        max_close_ts = int(now.timestamp() + (3 * 24 * 60 * 60))
+        min_close_ts = int(now.timestamp())
+
+        params = {
+            "status": "open",
+            "min_close_ts": min_close_ts,
+            "max_close_ts": max_close_ts,
+            "limit": 200,
+            "mve_filter": "exclude"
+        }
+
+        data = self._api_get("/markets", params)
+        markets = data.get("markets", [])
+
+        # Filter and process (without images)
+        valid_markets = []
+        event_ticker_to_market = {}  # For deduplication by event_ticker
+        
+        for market in markets:
+            title = market.get("title", "")
+            ticker = market.get("ticker", "")
+            category = market.get("category", "")
+            event_ticker = market.get("event_ticker", "")
+
+            if self._is_weather_market(title, ticker):
+                continue
+            if self._is_sports_market(title, ticker, category):
+                continue
+            if self._is_crypto_market(title, ticker):
+                continue
+
+            close_time = market.get("close_time", "")
+            if not close_time or not event_ticker:
+                continue
+
+            market_data = {
+                "title": title,
+                "ticker": ticker,
+                "event_ticker": event_ticker,
+                "slug": event_ticker.lower(),
+                "close_time": close_time,
+                "time_remaining": calculate_time_remaining(close_time),
+                "volume": market.get("volume", 0),
+                "volume_24h": market.get("volume_24h", 0),
+                "yes_bid": market.get("yes_bid", 0),
+                "yes_ask": market.get("yes_ask", 0),
+                "last_price": market.get("last_price", 0),
+                "liquidity": market.get("liquidity", 0),
+                "category": category
+            }
+
+            # Deduplicate by event_ticker - keep the one with highest volume_24h
+            if event_ticker not in event_ticker_to_market:
+                event_ticker_to_market[event_ticker] = market_data
+            else:
+                if market_data["volume_24h"] > event_ticker_to_market[event_ticker]["volume_24h"]:
+                    event_ticker_to_market[event_ticker] = market_data
+
+        # Convert dict values to list
+        valid_markets = list(event_ticker_to_market.values())
+
+        # Sort by volume_24h (highest first), then by close time (soonest first) as tiebreaker
+        valid_markets.sort(key=lambda x: (-x["volume_24h"], x["close_time"]))
+        
+        # Take top N and THEN fetch images
+        final_markets = valid_markets[:self.num_items]
+        final_markets = self._enrich_with_images(final_markets)
+
+        print(f"   Found {len(valid_markets)} total, returning top {len(final_markets)}")
+        return final_markets
+
+    def fetch_just_listed(self) -> List[Dict[str, Any]]:
+        """Fetch recently created markets - excludes sports, weather, crypto"""
+        print("Fetching newly listed markets...")
+
+        now = datetime.now(timezone.utc)
+        min_created_ts = int(now.timestamp() - (14 * 24 * 60 * 60))
+
+        params = {
+            "status": "open",
+            "min_created_ts": min_created_ts,
+            "limit": 200,
+            "mve_filter": "exclude"
+        }
+
+        data = self._api_get("/markets", params)
+        markets = data.get("markets", [])
+
+        # Filter and process (without images)
+        valid_markets = []
+        event_ticker_to_market = {}  # For deduplication by event_ticker
+        
+        for market in markets:
+            title = market.get("title", "")
+            ticker = market.get("ticker", "")
+            category = market.get("category", "")
+            event_ticker = market.get("event_ticker", "")
+
+            if self._is_weather_market(title, ticker):
+                continue
+            if self._is_sports_market(title, ticker, category):
+                continue
+            if self._is_crypto_market(title, ticker):
+                continue
+
+            created_time = market.get("created_time", "")
+            if not event_ticker:
+                continue
+
+            market_data = {
+                "title": title,
+                "ticker": ticker,
+                "event_ticker": event_ticker,
+                "slug": event_ticker.lower(),
+                "created_time": created_time,
+                "time_since": calculate_time_since(created_time),
+                "volume": market.get("volume", 0),
+                "volume_24h": market.get("volume_24h", 0),
+                "yes_bid": market.get("yes_bid", 0),
+                "yes_ask": market.get("yes_ask", 0),
+                "last_price": market.get("last_price", 0),
+                "liquidity": market.get("liquidity", 0),
+                "category": category
+            }
+
+            # Deduplicate by event_ticker - keep the one with highest volume_24h
+            if event_ticker not in event_ticker_to_market:
+                event_ticker_to_market[event_ticker] = market_data
+            else:
+                if market_data["volume_24h"] > event_ticker_to_market[event_ticker]["volume_24h"]:
+                    event_ticker_to_market[event_ticker] = market_data
+
+        # Convert dict values to list
+        valid_markets = list(event_ticker_to_market.values())
+
+        # Sort by volume_24h (highest first), then by created time (newest first) as tiebreaker
+        # Sort twice: first by created_time descending (newest first), then by volume_24h descending (stable sort preserves order)
+        valid_markets.sort(key=lambda x: x["created_time"], reverse=True)
+        valid_markets.sort(key=lambda x: x["volume_24h"], reverse=True)
+
+        # Take top N and THEN fetch images
+        final_markets = valid_markets[:self.num_items]
+        final_markets = self._enrich_with_images(final_markets)
+
+        print(f"   Found {len(valid_markets)} total, returning top {len(final_markets)}")
+        return final_markets
+
+    def fetch_biggest_movers(self) -> List[Dict[str, Any]]:
+        """
+        Fetch markets with the biggest price movements in the last 24h.
+        Uses last_price vs previous_price fields from Kalshi API.
+        """
+        print("Fetching biggest movers (24h price change)...")
+
+        markets = self._fetch_all_markets(status="open", limit=1000)
+
+        # Filter and calculate price changes (without images)
+        movers = []
+        event_ticker_to_market = {}  # For deduplication by event_ticker
+        
+        for market in markets:
+            title = market.get("title", "")
+            ticker = market.get("ticker", "")
+            category = market.get("category", "")
+            event_ticker = market.get("event_ticker", "")
+
+            if self._is_weather_market(title, ticker):
+                continue
+            if self._is_sports_market(title, ticker, category):
+                continue
+            if self._is_crypto_market(title, ticker):
+                continue
+
+            if not event_ticker:
+                continue
+
+            last_price = market.get("last_price", 0) or 0
+            previous_price = market.get("previous_price", 0) or 0
+            volume_24h = market.get("volume_24h", 0) or 0
+
+            if previous_price == 0 or last_price == 0:
+                continue
+            if volume_24h < 100:
+                continue
+
+            price_change = last_price - previous_price
+            price_change_pct = ((last_price - previous_price) / previous_price) * 100
+
+            if abs(price_change) < 3:
+                continue
+
+            market_data = {
+                "title": title,
+                "ticker": ticker,
+                "event_ticker": event_ticker,
+                "slug": event_ticker.lower(),
+                "current_price": last_price,
+                "previous_price": previous_price,
+                "price_change": price_change,
+                "price_change_pct": round(price_change_pct, 1),
+                "volume_24h": volume_24h,
+                "yes_bid": market.get("yes_bid", 0),
+                "yes_ask": market.get("yes_ask", 0),
+                "category": category
+            }
+
+            # Deduplicate by event_ticker - keep the one with highest absolute price change
+            if event_ticker not in event_ticker_to_market:
+                event_ticker_to_market[event_ticker] = market_data
+            else:
+                if abs(market_data["price_change"]) > abs(event_ticker_to_market[event_ticker]["price_change"]):
+                    event_ticker_to_market[event_ticker] = market_data
+
+        # Convert dict values to list
+        movers = list(event_ticker_to_market.values())
+
+        # Sort by absolute price change (biggest movers first)
+        movers.sort(key=lambda x: abs(x["price_change"]), reverse=True)
+
+        # Take top N and THEN fetch images
+        final_markets = movers[:self.num_items]
+        final_markets = self._enrich_with_images(final_markets)
+
+        print(f"   Found {len(movers)} total, returning top {len(final_markets)}")
+        return final_markets
+
+    def fetch_hot_markets(self) -> List[Dict[str, Any]]:
+        """Fetch markets with highest 24h volume - excludes weather"""
+        print("Fetching hot markets (by 24h volume)...")
+
+        markets = self._fetch_all_markets(status="open", limit=1000)
+
+        # Filter by volume (without images)
+        hot_markets = []
+        event_ticker_to_market = {}  # For deduplication by event_ticker
+        
+        for market in markets:
+            title = market.get("title", "")
+            ticker = market.get("ticker", "")
+            category = market.get("category", "")
+            event_ticker = market.get("event_ticker", "")
+
+            if self._is_weather_market(title, ticker):
+                continue
+
+            if not event_ticker:
+                continue
+
+            volume_24h = market.get("volume_24h", 0) or 0
+            if volume_24h < 100:
+                continue
+
+            market_data = {
+                "title": title,
+                "ticker": ticker,
+                "event_ticker": event_ticker,
+                "slug": event_ticker.lower(),
+                "volume_24h": volume_24h,
+                "volume": market.get("volume", 0),
+                "yes_bid": market.get("yes_bid", 0),
+                "yes_ask": market.get("yes_ask", 0),
+                "last_price": market.get("last_price", 0),
+                "liquidity": market.get("liquidity", 0),
+                "category": category
+            }
+
+            # Deduplicate by event_ticker - keep the one with highest volume_24h
+            if event_ticker not in event_ticker_to_market:
+                event_ticker_to_market[event_ticker] = market_data
+            else:
+                if market_data["volume_24h"] > event_ticker_to_market[event_ticker]["volume_24h"]:
+                    event_ticker_to_market[event_ticker] = market_data
+
+        # Convert dict values to list
+        hot_markets = list(event_ticker_to_market.values())
+
+        # Sort by 24h volume (highest first)
+        hot_markets.sort(key=lambda x: x["volume_24h"], reverse=True)
+
+        # Take top N and THEN fetch images
+        final_markets = hot_markets[:self.num_items]
+        final_markets = self._enrich_with_images(final_markets)
+
+        print(f"   Found {len(hot_markets)} total, returning top {len(final_markets)}")
+        return final_markets
+
+    def fetch_sports_events(self) -> List[Dict[str, Any]]:
+        """Fetch sports-related markets"""
+        print("Fetching sports events...")
+
+        markets = self._fetch_all_markets(status="open", limit=1000)
+
+        # Filter for sports (without images)
+        sports_markets = []
+        event_ticker_to_market = {}  # For deduplication by event_ticker
+        
+        for market in markets:
+            title = market.get("title", "")
+            ticker = market.get("ticker", "")
+            category = market.get("category", "")
+            event_ticker = market.get("event_ticker", "")
+
+            if not self._is_sports_market(title, ticker, category):
+                continue
+
+            if not event_ticker:
+                continue
+
+            # Determine sport type
+            sport = "Sports"
+            title_lower = title.lower()
+            if any(x in title_lower for x in ["nfl", "football", "packers", "bears", "cowboys", "patriots", "super bowl"]):
+                sport = "NFL"
+            elif any(x in title_lower for x in ["nba", "basketball", "lakers", "celtics", "warriors"]):
+                sport = "NBA"
+            elif any(x in title_lower for x in ["mlb", "baseball", "yankees", "dodgers"]):
+                sport = "MLB"
+            elif any(x in title_lower for x in ["nhl", "hockey"]):
+                sport = "NHL"
+            elif any(x in title_lower for x in ["ufc", "mma", "fight", "boxing"]):
+                sport = "UFC/MMA"
+            elif any(x in title_lower for x in ["soccer", "premier league", "champions league", "world cup"]):
+                sport = "Soccer"
+            elif "golf" in title_lower:
+                sport = "Golf"
+            elif any(x in title_lower for x in ["tennis", "wimbledon"]):
+                sport = "Tennis"
+            elif any(x in title_lower for x in ["f1", "racing", "nascar"]):
+                sport = "Racing"
+
+            market_data = {
+                "title": title,
+                "ticker": ticker,
+                "event_ticker": event_ticker,
+                "slug": event_ticker.lower(),
+                "close_time": market.get("close_time", ""),
+                "time_remaining": calculate_time_remaining(market.get("close_time", "")),
+                "volume": market.get("volume", 0),
+                "volume_24h": market.get("volume_24h", 0),
+                "yes_bid": market.get("yes_bid", 0),
+                "yes_ask": market.get("yes_ask", 0),
+                "last_price": market.get("last_price", 0),
+                "sport": sport
+            }
+
+            # Deduplicate by event_ticker - keep the one with highest volume_24h
+            if event_ticker not in event_ticker_to_market:
+                event_ticker_to_market[event_ticker] = market_data
+            else:
+                if market_data["volume_24h"] > event_ticker_to_market[event_ticker]["volume_24h"]:
+                    event_ticker_to_market[event_ticker] = market_data
+
+        # Convert dict values to list
+        sports_markets = list(event_ticker_to_market.values())
+
+        # Sort by 24h volume (most active first)
+        sports_markets.sort(key=lambda x: x.get("volume_24h", 0), reverse=True)
+
+        # Take top N and THEN fetch images
+        final_markets = sports_markets[:self.num_items]
+        final_markets = self._enrich_with_images(final_markets)
+
+        print(f"   Found {len(sports_markets)} total, returning top {len(final_markets)}")
+        return final_markets
+
+    def display_menu(self) -> None:
+        """Display the menu of available markets"""
+        self.build_menu(verbose=True)
+
+    def build_menu(self, verbose: bool = False) -> List[Dict[str, Any]]:
+        """Populate structured menu entries and optionally print them."""
+        sections = [
+            ("ending_soon", "Ending Soon", self.fetch_ending_soon, "⏰"),
+            ("just_listed", "New", self.fetch_just_listed, "🆕"),
+            ("biggest_movers", "Biggest Movers", self.fetch_biggest_movers, "📈"),
+            ("hot_markets", "Hot Markets", self.fetch_hot_markets, "🔥"),
+            ("sports_events", "Top Sports", self.fetch_sports_events, "🏆"),
+        ]
+
+        if verbose:
+            print("\n" + "=" * 60)
+            print("KALSHI DATA MENU")
+            print("=" * 60)
+
+        menu_entries: List[Dict[str, Any]] = []
+        counter = 1
+        self.menu_items = {}
+
+        for key, label, fetcher, emoji in sections:
+            items = fetcher()
+
+            if verbose:
+                print(f"\n{emoji} {label.upper()}")
+                print("-" * 40)
+
+            for item in items:
+                self.menu_items[counter] = (key, item)
+                url = f"https://kalshi.com/markets/{item.get('event_ticker', '').lower()}"
+
+                menu_entries.append({
+                    "id": counter,
+                    "category": key,
+                    "label": label,
+                    "emoji": emoji,
+                    "data": item,
+                    "url": url,
+                })
+
+                if verbose:
+                    title = item.get("title", "Untitled")
+                    print(f"{counter}. {title}")
+
+                    if key == "biggest_movers":
+                        change = item.get("price_change", 0)
+                        change_pct = item.get("price_change_pct", 0)
+                        sign = "+" if change > 0 else ""
+                        print(f"   {sign}{change}¢ ({sign}{change_pct}%) | Vol 24h: {item.get('volume_24h', 0):,}")
+                    elif key == "ending_soon":
+                        print(f"   Ends: {item.get('time_remaining', 'Soon')} | Price: {item.get('last_price', 0)}¢")
+                    elif key == "just_listed":
+                        print(f"   Listed: {item.get('time_since', 'Recently')} | Price: {item.get('last_price', 0)}¢")
+                    elif key == "hot_markets":
+                        print(f"   Vol 24h: {item.get('volume_24h', 0):,} | Price: {item.get('last_price', 0)}¢")
+                    elif key == "sports_events":
+                        print(f"   {item.get('sport', 'Sports')} | {item.get('time_remaining', 'Soon')}")
+
+                counter += 1
+
+        if verbose:
+            print("\n" + "=" * 60)
+
+        return menu_entries
+
+    def get_user_selections(self) -> List[tuple]:
+        """Get user's market selections with optional 's' suffix for sports section"""
+        print("\n>>> Enter Your Selections (comma-separated numbers):")
+        print("    Example: 2, 3, 4, 9, 10, 35")
+        print("    Add 's' to move to Sports section: 21s, 25s")
+        print("    Or press Enter to skip selection")
+        print("    Or input 'z' to go back\n")
+
+        user_input = input(">>> ").strip()
+
+        if user_input.lower() == 'z':
+            return 'BACK'
+
+        if not user_input:
+            return []
+
+        try:
+            valid_selections = []
+            for item in user_input.split(','):
+                item = item.strip()
+
+                force_sports = False
+                if item.endswith('s') or item.endswith('S'):
+                    force_sports = True
+                    item = item[:-1]
+
+                sel = int(item)
+                if sel in self.menu_items:
+                    valid_selections.append((sel, force_sports))
+                else:
+                    print(f"Warning: Selection {sel} is invalid (out of range)")
+
+            return valid_selections
+        except ValueError:
+            print("Error: Invalid input. Please enter comma-separated numbers.")
+            return self.get_user_selections()
+
+    def organize_selections(self, selections: List[tuple]) -> Dict[str, List[Dict]]:
+        """Organize selected markets by category with sports override support"""
+        organized = defaultdict(list)
+
+        for item in selections:
+            if isinstance(item, tuple):
+                sel, force_sports = item
+            else:
+                sel = item
+                force_sports = False
+
+            category, market_data = self.menu_items[sel]
+
+            if force_sports and category != "sports_events":
+                print(f"   Moving item {sel} from {category} to sports section")
+                category = "sports_events"
+
+            organized[category].append(market_data)
+
+        return organized
+
+    def export_json(self, filename: str = "kalshi_data.json") -> None:
+        """Export all fetched data to JSON"""
+        menu_entries = self.build_menu(verbose=False)
+
+        output = {
+            "generated_at": datetime.now(timezone.utc).isoformat(),
+            "source": "Kalshi API",
+            "sections": {}
+        }
+
+        for entry in menu_entries:
+            category = entry["category"]
+            if category not in output["sections"]:
+                output["sections"][category] = {
+                    "label": entry["label"],
+                    "emoji": entry["emoji"],
+                    "items": []
+                }
+            output["sections"][category]["items"].append(entry["data"])
+
+        with open(filename, 'w') as f:
+            json.dump(output, f, indent=2)
+
+        print(f"\n✅ Data exported to {filename}")
+
+    def run(self):
+        """Main execution flow"""
+        print("\nStarting Kalshi Data Collection")
+        print(f"   Items per section: {self.num_items}")
+
+        self.display_menu()
+        selections = self.get_user_selections()
+
+        if selections == 'BACK':
+            print("\n⚠️  Can't go back - this is the first step")
+            return
+
+        if not selections:
+            print("\nWarning: No selections made. Exiting.")
+            return
+
+        print(f"\nSelected items: {[s[0] if isinstance(s, tuple) else s for s in selections]}")
+        organized = self.organize_selections(selections)
+
+        print("\n✅ Data collection complete!")
+        print(f"   Total sections: {len(organized)}")
+        print(f"   Total items: {len(selections)}")
+
+        for category, items in organized.items():
+            print(f"   • {category}: {len(items)} items")
+
+        print(f"\n🎉 Done! Data is ready for use.")
+
+
+def main():
+    """Main entry point"""
+    parser = argparse.ArgumentParser(description='Collect Kalshi Market Data')
+    parser.add_argument(
+        'num_items',
+        type=int,
+        nargs='?',
+        default=10,
+        help='Number of items per section (default: 10)'
+    )
+    parser.add_argument(
+        '--selections',
+        type=str,
+        help='Comma-separated list of selections (e.g., "1,3,5,7,10")'
+    )
+    parser.add_argument(
+        '--export',
+        type=str,
+        help='Export all data to JSON file (e.g., --export data.json)'
+    )
+    parser.add_argument(
+        '--no-interactive',
+        action='store_true',
+        help='Run without interactive selection (use with --export)'
+    )
+
+    args = parser.parse_args()
+
+    if args.num_items < 1:
+        print("Error: Number of items must be at least 1")
+        sys.exit(1)
+
+    if args.num_items > 50:
+        print("Warning: Large number of items may slow down API requests")
+
+    generator = KalshiDataGenerator(num_items=args.num_items)
+
+    if args.export:
+        generator.export_json(args.export)
+        return
+
+    if args.selections:
+        generator.display_menu()
+
+        valid_selections = []
+        for item in args.selections.split(','):
+            item = item.strip()
+            force_sports = False
+            if item.endswith('s') or item.endswith('S'):
+                force_sports = True
+                item = item[:-1]
+
+            try:
+                sel = int(item)
+                if sel in generator.menu_items:
+                    valid_selections.append((sel, force_sports))
+                else:
+                    print(f"Warning: Selection {sel} is invalid")
+            except ValueError:
+                print(f"Warning: Invalid selection '{item}'")
+
+        if valid_selections:
+            print(f"\nUsing command-line selections: {[s[0] for s in valid_selections]}")
+            organized = generator.organize_selections(valid_selections)
+
+            print("\n✅ Data collection complete!")
+            print(f"   Total sections: {len(organized)}")
+            print(f"   Total items: {len(valid_selections)}")
+            for category, items in organized.items():
+                print(f"   • {category}: {len(items)} items")
+        return
+
+    if not args.no_interactive:
+        generator.run()
+
+
+if __name__ == "__main__":
+    main()

app/main.py

@@ -16,7 +16,7 @@ from .post_process import convert_polymarket_links, minify_html
 from .schemas import TemplateifyResponse
 from .template_manager import TemplateManager
 from .templateify_new_service import TemplateifyNewService
-from .email3 import PolymarketEmailGenerator
+from .kalshi_data import KalshiDataGenerator
 
 logger = logging.getLogger("template-manager")
 
@@ -378,32 +378,29 @@ async def post_note_to_substack(
 async def fetch_menu_data(
     request: dict = Body(...),
 ) -> dict:
-    """Fetch menu data from Polymarket APIs using curl_cffi.
+    """Fetch menu data from Kalshi API.
     
     Request body:
         num_items: Number of items per section (default: 10)
-        skip_dub: Skip creating Dub tracking links (default: False)
     """
     try:
         num_items = request.get("num_items", 10)
-        skip_dub = request.get("skip_dub", False)
         
-        logger.info(f"Fetching menu data (num_items={num_items}, skip_dub={skip_dub})...")
+        logger.info(f"Fetching menu data from Kalshi API (num_items={num_items})...")
         
         # Create generator instance
-        generator = PolymarketEmailGenerator(
-            num_items=num_items,
-            skip_dub=skip_dub,
-        )
+        generator = KalshiDataGenerator(num_items=num_items)
         
-        # Build menu (this fetches all data including breaking news)
+        # Build menu (this fetches all data)
         logger.info("Building menu data...")
-        menu_entries = generator.build_menu(verbose=True)
+        menu_entries = generator.build_menu(verbose=False)
         
         # Format as MenuResponse structure
-        from datetime import datetime
+        from datetime import datetime, timezone
         menu_response = {
-            "generated_at": datetime.utcnow().isoformat(),
+            "generated_at": datetime.now(timezone.utc).isoformat(),
+            "source": "Kalshi API",
+            "api_base": "https://api.elections.kalshi.com/trade-api/v2",
             "items": menu_entries,
         }
         

app/mini_converter.py

@@ -32,7 +32,7 @@ class EmailConverter:
     def __init__(self, settings: Settings):
         self.settings = settings
         self.template_dir = Path(settings.template_root).resolve()
-        self.supabase_base_url = "https://tyknilsxnmoujlfwpdge.supabase.co/storage/v1/object/public/Polygrapher"
+        self.supabase_base_url = "https://tyknilsxnmoujlfwpdge.supabase.co/storage/v1/object/public/Kalshifier"
         self.header_image_width = None  # Will be set when processing header image
         
         # Initialize Cloudinary if configured

app/storage_client.py

@@ -11,7 +11,7 @@ class TemplateStorageClient:
     def __init__(self, url: Optional[str], key: Optional[str], prefix: str = "") -> None:
         self.url = url
         self.key = key
-        self.bucket = "Polygrapher"
+        self.bucket = "Kalshifier"
         self.logger = logging.getLogger("template-storage")
 
     @property