data_utils.py

"""
Enhanced data handlers for multiple geospatial data sources

External APIs integrated:
- OpenStreetMap Overpass API: Geographic features (POIs, roads, natural features)
- GLOBIL/ArcGIS Hub: WWF conservation datasets
"""
import pandas as pd
import requests
from typing import Dict, List, Optional, Tuple, Any
import json
import logging

# Import external API clients
try:
    from external_apis import (
        ExternalDataFetcher,
        OverpassAPIClient,
        GlobILClient,
        COUNTRY_ISO_CODES,
        OSM_POI_CATEGORIES,
        GLOBIL_TOPICS,
        get_available_apis
    )
    EXTERNAL_APIS_AVAILABLE = True
except ImportError:
    EXTERNAL_APIS_AVAILABLE = False
    logging.warning("External APIs module not found. Extended features disabled.")

logger = logging.getLogger(__name__)


class DataEnhancer:
    """
    Additional data sources and enrichment for geospatial queries
    """
    
    @staticmethod
    def get_sample_economic_data():
        """
        Sample economic indicators (in production, connect to World Bank API)
        """
        return {
            'United States': {'gdp_growth': 2.1, 'unemployment': 3.7, 'inflation': 3.2},
            'China': {'gdp_growth': 5.2, 'unemployment': 5.0, 'inflation': 0.2},
            'Germany': {'gdp_growth': 0.1, 'unemployment': 3.0, 'inflation': 6.1},
            'India': {'gdp_growth': 7.2, 'unemployment': 8.0, 'inflation': 5.4},
            'Brazil': {'gdp_growth': 2.9, 'unemployment': 8.5, 'inflation': 4.6},
            'United Kingdom': {'gdp_growth': 0.5, 'unemployment': 3.9, 'inflation': 4.0},
            'France': {'gdp_growth': 0.9, 'unemployment': 7.2, 'inflation': 5.2},
            'Japan': {'gdp_growth': 1.9, 'unemployment': 2.6, 'inflation': 3.2},
            'South Korea': {'gdp_growth': 1.4, 'unemployment': 2.7, 'inflation': 3.6},
            'Canada': {'gdp_growth': 1.1, 'unemployment': 5.4, 'inflation': 3.9}
        }
    
    @staticmethod
    def get_sample_environmental_data():
        """
        Sample environmental indicators
        """
        return {
            'United States': {'co2_per_capita': 15.5, 'renewable_energy': 12.6, 'forest_coverage': 33.9},
            'China': {'co2_per_capita': 7.4, 'renewable_energy': 12.4, 'forest_coverage': 23.0},
            'Germany': {'co2_per_capita': 8.4, 'renewable_energy': 19.3, 'forest_coverage': 32.7},
            'India': {'co2_per_capita': 1.9, 'renewable_energy': 17.5, 'forest_coverage': 24.4},
            'Brazil': {'co2_per_capita': 2.2, 'renewable_energy': 46.1, 'forest_coverage': 59.4},
            'Russia': {'co2_per_capita': 11.4, 'renewable_energy': 5.1, 'forest_coverage': 49.8},
            'Japan': {'co2_per_capita': 8.7, 'renewable_energy': 10.2, 'forest_coverage': 68.5},
            'Australia': {'co2_per_capita': 16.8, 'renewable_energy': 11.9, 'forest_coverage': 17.4}
        }
    
    @staticmethod
    def enrich_dataframe(df: pd.DataFrame, data_type: str = 'economic') -> pd.DataFrame:
        """
        Enrich existing dataframe with additional indicators
        """
        enriched_df = df.copy()
        
        if data_type == 'economic':
            extra_data = DataEnhancer.get_sample_economic_data()
        elif data_type == 'environmental':
            extra_data = DataEnhancer.get_sample_environmental_data()
        else:
            return enriched_df
        
        # Add new columns
        for indicator in ['gdp_growth', 'unemployment', 'inflation',
                            'co2_per_capita', 'renewable_energy', 'forest_coverage']:
            enriched_df[indicator] = enriched_df['name'].map(
                lambda x: extra_data.get(x, {}).get(indicator, None)
            )
        
        return enriched_df
    
    @staticmethod
    def get_regional_aggregates(df: pd.DataFrame) -> pd.DataFrame:
        """
        Calculate regional aggregates
        """
        regional_stats = df.groupby('continent').agg({
            'pop_est': 'sum',
            'gdp_md_est': 'sum',
            'name': 'count'
        }).reset_index()
        
        regional_stats.columns = ['continent', 'total_population', 'total_gdp', 'country_count']
        regional_stats['avg_gdp_per_capita'] = (
            regional_stats['total_gdp'] / regional_stats['total_population'] * 1000000
        )
        
        return regional_stats

class QueryEnhancer:
    """
    Enhance and validate queries
    """
    
    CONTINENT_MAP = {
        'asia': 'Asia',
        'europe': 'Europe',
        'africa': 'Africa',
        'north america': 'North America',
        'south america': 'South America',
        'oceania': 'Oceania',
        'antarctica': 'Antarctica'
    }
    
    COUNTRY_GROUPS = {
        'brics': ['Brazil', 'Russia', 'India', 'China', 'South Africa'],
        'g7': ['United States of America', 'Japan', 'Germany', 'United Kingdom', 
                'France', 'Italy', 'Canada'],
        'asean': ['Indonesia', 'Thailand', 'Philippines', 'Vietnam', 'Myanmar',
                    'Malaysia', 'Singapore', 'Cambodia', 'Laos', 'Brunei'],
        'gcc': ['Saudi Arabia', 'United Arab Emirates', 'Kuwait', 'Qatar', 'Bahrain', 'Oman'],
        'eu': ['Germany', 'France', 'Italy', 'Spain', 'Poland', 'Romania', 'Netherlands',
                'Belgium', 'Greece', 'Portugal', 'Czech Republic', 'Hungary', 'Sweden',
                'Austria', 'Bulgaria', 'Denmark', 'Finland', 'Slovakia', 'Ireland',
                'Croatia', 'Lithuania', 'Slovenia', 'Latvia', 'Estonia', 'Cyprus', 
                'Luxembourg', 'Malta']
    }
    
    @classmethod
    def expand_location(cls, location: str) -> List[str]:
        """
        Expand location strings to actual country/region names
        """
        location_lower = location.lower()
        
        # Check if it's a continent
        if location_lower in cls.CONTINENT_MAP:
            return [cls.CONTINENT_MAP[location_lower]]
        
        # Check if it's a country group
        if location_lower in cls.COUNTRY_GROUPS:
            return cls.COUNTRY_GROUPS[location_lower]
        
        # Return as-is
        return [location]
    
    @classmethod
    def validate_indicators(cls, indicators: List[str]) -> List[str]:
        """
        Validate and normalize indicator names
        """
        valid_indicators = []
        indicator_mapping = {
            'population': 'pop_est',
            'gdp': 'gdp_md_est',
            'density': 'pop_density',
            'per capita': 'gdp_per_capita',
            'co2': 'co2_per_capita',
            'renewable': 'renewable_energy',
            'forest': 'forest_coverage',
            'growth': 'gdp_growth',
            'unemployment': 'unemployment',
            'inflation': 'inflation'
        }
        
        for indicator in indicators:
            indicator_lower = indicator.lower()
            for key, value in indicator_mapping.items():
                if key in indicator_lower:
                    valid_indicators.append(value)
                    break
            else:
                valid_indicators.append('pop_est')  # default
        
        return list(set(valid_indicators))  # Remove duplicates

# Statistical analysis utilities
class GeoStats:
    """
    Statistical analysis for geospatial data
    """
    
    @staticmethod
    def calculate_correlation(df: pd.DataFrame, col1: str, col2: str) -> float:
        """
        Calculate correlation between two indicators
        """
        try:
            return df[[col1, col2]].corr().iloc[0, 1]
        except:
            return 0.0
    
    @staticmethod
    def get_outliers(df: pd.DataFrame, column: str) -> pd.DataFrame:
        """
        Identify outliers using IQR method
        """
        Q1 = df[column].quantile(0.25)
        Q3 = df[column].quantile(0.75)
        IQR = Q3 - Q1
        
        lower_bound = Q1 - 1.5 * IQR
        upper_bound = Q3 + 1.5 * IQR
        
        outliers = df[(df[column] < lower_bound) | (df[column] > upper_bound)]
        return outliers
    
    @staticmethod
    def generate_summary_stats(df: pd.DataFrame, column: str) -> Dict:
        """
        Generate summary statistics for a column
        """
        return {
            'mean': df[column].mean(),
            'median': df[column].median(),
            'std': df[column].std(),
            'min': df[column].min(),
            'max': df[column].max(),
            'count': df[column].count()
        }


# =============================================================================
# External Data Handler - Unified interface for external APIs
# =============================================================================

class ExternalDataHandler:
    """
    Unified handler for external geographic and environmental data APIs.
    
    Provides easy access to:
    - OpenStreetMap Overpass API (POIs, environmental features)
    - GLOBIL/WWF conservation datasets
    """
    
    def __init__(self):
        self._fetcher = None
        self._overpass = None
        self._globil = None
        self._initialized = False
        
    def _ensure_initialized(self):
        """Lazy initialization of API clients."""
        if self._initialized:
            return
        
        if not EXTERNAL_APIS_AVAILABLE:
            logger.error("External APIs module not available")
            return
        
        try:
            self._fetcher = ExternalDataFetcher()
            self._overpass = OverpassAPIClient()
            self._globil = GlobILClient()
            self._initialized = True
        except Exception as e:
            logger.error(f"Failed to initialize external APIs: {e}")
    
    @property
    def is_available(self) -> bool:
        """Check if external APIs are available."""
        return EXTERNAL_APIS_AVAILABLE
    
    # =========================================================================
    # OpenStreetMap Overpass API Methods
    # =========================================================================
    
    def get_pois_in_area(
        self,
        bbox: Tuple[float, float, float, float],
        category: str = "Tourism",
        limit: int = 50
    ) -> List[Dict]:
        """
        Get Points of Interest in a bounding box.
        
        Args:
            bbox: (south, west, north, east) coordinates
            category: POI category from OSM_POI_CATEGORIES
            limit: Maximum results per sub-category
            
        Returns:
            List of POI dictionaries with name, type, coordinates
        """
        self._ensure_initialized()
        if not self._overpass:
            return []
        
        if not EXTERNAL_APIS_AVAILABLE:
            return []
        
        poi_config = OSM_POI_CATEGORIES.get(category, {})
        tag = poi_config.get("tag", "amenity")
        values = poi_config.get("values", [])
        
        all_pois = []
        for value in values[:5]:
            pois = self._overpass.get_pois_in_bbox(
                bbox[0], bbox[1], bbox[2], bbox[3],
                poi_type=tag,
                poi_value=value,
                limit=limit
            )
            all_pois.extend(pois)
        
        return all_pois
    
    def get_pois_near_location(
        self,
        lat: float,
        lon: float,
        radius_km: float = 5,
        category: str = "Tourism"
    ) -> List[Dict]:
        """
        Get Points of Interest near a location.
        
        Args:
            lat, lon: Center point coordinates
            radius_km: Search radius in kilometers
            category: POI category
            
        Returns:
            List of POI dictionaries
        """
        self._ensure_initialized()
        if not self._overpass:
            return []
        
        if not EXTERNAL_APIS_AVAILABLE:
            return []
        
        poi_config = OSM_POI_CATEGORIES.get(category, {})
        tag = poi_config.get("tag", "amenity")
        values = poi_config.get("values", [])
        
        # Use smaller radius and limit queries to avoid rate limiting
        radius_meters = min(int(radius_km * 1000), 3000)  # Max 3km radius
        all_pois = []
        
        # Only query first 3 POI types to reduce API calls
        for value in values[:3]:
            pois = self._overpass.get_pois_around_point(
                lat, lon, radius_meters,
                poi_type=tag,
                poi_value=value
            )
            all_pois.extend(pois)
            # Stop if we have enough POIs
            if len(all_pois) >= 50:
                break
        
        return all_pois[:50]  # Cap total results
    
    def get_natural_features(
        self,
        bbox: Tuple[float, float, float, float],
        feature_types: List[str] = None
    ) -> Dict[str, List[Dict]]:
        """
        Get natural/environmental features from OSM.
        
        Args:
            bbox: (south, west, north, east) coordinates
            feature_types: List of types ('natural', 'forest', 'park', 'protected_area', 'water')
            
        Returns:
            Dictionary mapping feature type to list of features
        """
        self._ensure_initialized()
        if not self._overpass:
            return {}
        
        if feature_types is None:
            feature_types = ["natural", "forest", "park", "protected_area"]
        
        results = {}
        for ftype in feature_types:
            features = self._overpass.get_environmental_features(
                bbox[0], bbox[1], bbox[2], bbox[3],
                feature_type=ftype
            )
            results[ftype] = features
        
        return results
    
    @staticmethod
    def get_poi_categories() -> Dict[str, Dict]:
        """Get available POI categories."""
        if EXTERNAL_APIS_AVAILABLE:
            return OSM_POI_CATEGORIES
        return {}
    
    # =========================================================================
    # GLOBIL/WWF Conservation Data Methods
    # =========================================================================
    
    def search_conservation_data(
        self,
        topic: str,
        limit: int = 10
    ) -> List[Dict]:
        """
        Search WWF GLOBIL for conservation datasets.
        
        Args:
            topic: Search term (e.g., 'deforestation', 'endangered species')
            limit: Maximum number of results
            
        Returns:
            List of dataset metadata dictionaries
        """
        self._ensure_initialized()
        if not self._globil:
            return []
        
        return self._globil.search_datasets(topic, max_results=limit)
    
    def get_conservation_topics(self) -> Dict[str, str]:
        """Get available conservation data topics."""
        if EXTERNAL_APIS_AVAILABLE:
            return GLOBIL_TOPICS
        return {}
    
    def get_public_conservation_layers(self, category: str = "forests") -> List[Dict]:
        """
        Get publicly accessible conservation data layers.
        
        Args:
            category: Topic category ('forests', 'wildlife', 'oceans', 'freshwater', 'climate')
            
        Returns:
            List of accessible layer metadata
        """
        self._ensure_initialized()
        if not self._globil:
            return []
        
        return self._globil.get_public_feature_layers(category)

    def fetch_conservation_features(
        self,
        topic: str,
        max_datasets: int = 3,
        max_features: int = 500
    ) -> List[Dict]:
        """
        Fetch actual feature data from conservation datasets.
        
        Args:
            topic: Conservation topic to search
            max_datasets: Maximum datasets to query
            max_features: Maximum features per dataset
            
        Returns:
            List of dictionaries with dataset info and GeoJSON features
        """
        self._ensure_initialized()
        if not self._globil:
            return []
        
        return self._globil.search_and_fetch_features(
            topic,
            max_datasets=max_datasets,
            max_features_per_dataset=max_features
        )

    def get_conservation_dataset_features(
        self,
        item_id: str,
        max_features: int = 1000
    ) -> Optional[Dict]:
        """
        Get features from a specific conservation dataset.
        
        Args:
            item_id: ArcGIS item ID
            max_features: Maximum features to return
            
        Returns:
            Dictionary with dataset info and GeoJSON features
        """
        self._ensure_initialized()
        if not self._globil:
            return None
        
        return self._globil.get_dataset_with_features(item_id, max_features)

    # =========================================================================
    # Utility Methods
    # =========================================================================
    
    @staticmethod
    def get_country_code(country_name: str) -> Optional[str]:
        """Get ISO code for a country name."""
        if EXTERNAL_APIS_AVAILABLE:
            return COUNTRY_ISO_CODES.get(country_name)
        return None
    
    def get_api_status(self) -> Dict[str, Any]:
        """
        Get status information about all external APIs.
        
        Returns:
            Dictionary with API status information
        """
        if EXTERNAL_APIS_AVAILABLE:
            return get_available_apis()
        return {
            "status": "unavailable",
            "message": "External APIs module not loaded"
        }
    
    def get_all_country_codes(self) -> Dict[str, str]:
        """Get mapping of country names to ISO codes."""
        if EXTERNAL_APIS_AVAILABLE:
            return COUNTRY_ISO_CODES
        return {}


# Create a singleton instance for easy access
external_data = ExternalDataHandler()