feat: Add map and chart visualizations for conservation data

- Add GlobILClient.fetch_features() to get actual GeoJSON features from Feature Services
- Add GlobILClient.search_and_fetch_features() for topic-based feature search
- Add GlobILClient.get_dataset_with_features() for specific dataset features
- Add ExternalDataHandler.fetch_conservation_features() wrapper
- Add create_conservation_map() for interactive Folium maps with polygon/point layers
- Add create_conservation_chart() for bar/pie charts of feature distribution
- Update process_query() conservation routing to generate maps and charts
- Improve city name matching for POI queries (Berlin, Germany now works)
- Add logger to app.py for consistent logging

app.py

@@ -15,6 +15,11 @@ import numpy as np
 from pathlib import Path
 import warnings
 import logging
+import sys
+
+# Suppress Python 3.13 asyncio cleanup warnings (harmless garbage collection issue)
+if sys.version_info >= (3, 13):
+    warnings.filterwarnings('ignore', message='.*Invalid file descriptor.*')
 
 # Suppress GeoPandas CRS warnings (area/centroid calculations are approximate for demo purposes)
 warnings.filterwarnings('ignore', message='.*Geometry is in a geographic CRS.*')
@@ -23,6 +28,9 @@ warnings.filterwarnings('ignore', message='.*Geometry is in a geographic CRS.*')
 warnings.filterwarnings('ignore', message='.*Invalid file descriptor.*')
 logging.getLogger('asyncio').setLevel(logging.CRITICAL)
 
+# Configure logger for this module
+logger = logging.getLogger(__name__)
+
 import branca.colormap as cm
 import zipfile
 import urllib.request
@@ -241,15 +249,21 @@ def fetch_poi_data(parsed_query):
     city_coords = None
     city_name = None
     for loc in locations:
-        loc_lower = loc.lower().replace(",", "").strip()
-        # Check direct match
+        # Normalize: lowercase, remove commas, extra spaces
+        loc_lower = loc.lower().replace(",", " ").strip()
+        loc_words = set(loc_lower.split())
+        
+        # Check direct match first
         if loc_lower in CITY_COORDINATES:
             city_coords = CITY_COORDINATES[loc_lower]
             city_name = loc
             break
-        # Check partial match
+        
+        # Check if any city name appears in the location string
         for city, coords in CITY_COORDINATES.items():
-            if city in loc_lower or loc_lower in city:
+            city_words = set(city.split())
+            # Match if city name is contained in location OR location word matches city
+            if city in loc_lower or city_words & loc_words:
                 city_coords = coords
                 city_name = loc
                 break
@@ -393,6 +407,250 @@ def create_conservation_table(datasets):
     return pd.DataFrame(rows)
 
 
+def create_conservation_map(feature_data_list, map_style='Light'):
+    """
+    Create a Folium map showing conservation features from GLOBIL.
+    
+    Args:
+        feature_data_list: List of dicts with 'dataset' and 'features' keys
+        map_style: Map tile style
+        
+    Returns:
+        Folium map object
+    """
+    if not feature_data_list:
+        return None
+    
+    # Collect all coordinates to calculate center
+    all_coords = []
+    
+    for data in feature_data_list:
+        for feature in data.get('features', []):
+            geom = feature.get('geometry', {})
+            geom_type = geom.get('type', '')
+            coords = geom.get('coordinates', [])
+            
+            if geom_type == 'Point' and len(coords) >= 2:
+                all_coords.append((coords[1], coords[0]))  # lat, lon
+            elif geom_type == 'Polygon' and coords:
+                # Get centroid of first ring
+                ring = coords[0] if isinstance(coords[0], list) and coords[0] else []
+                if ring and len(ring) > 0:
+                    lons = [c[0] for c in ring if len(c) >= 2]
+                    lats = [c[1] for c in ring if len(c) >= 2]
+                    if lons and lats:
+                        all_coords.append((sum(lats)/len(lats), sum(lons)/len(lons)))
+            elif geom_type == 'MultiPolygon' and coords:
+                for polygon in coords:
+                    if polygon and polygon[0]:
+                        ring = polygon[0]
+                        lons = [c[0] for c in ring if len(c) >= 2]
+                        lats = [c[1] for c in ring if len(c) >= 2]
+                        if lons and lats:
+                            all_coords.append((sum(lats)/len(lats), sum(lons)/len(lons)))
+    
+    if not all_coords:
+        # Default to world center if no coordinates found
+        center_lat, center_lon = 20, 0
+    else:
+        center_lat = sum(c[0] for c in all_coords) / len(all_coords)
+        center_lon = sum(c[1] for c in all_coords) / len(all_coords)
+    
+    tiles = MAP_STYLES.get(map_style, 'CartoDB positron')
+    m = folium.Map(location=[center_lat, center_lon], zoom_start=3, tiles=tiles)
+    
+    # Color palette for different datasets
+    colors = ['#2ecc71', '#3498db', '#9b59b6', '#e74c3c', '#f39c12', '#1abc9c']
+    
+    for i, data in enumerate(feature_data_list):
+        dataset = data.get('dataset', {})
+        features = data.get('features', [])
+        dataset_title = dataset.get('title', f'Dataset {i+1}')
+        color = colors[i % len(colors)]
+        
+        # Create a feature group for this dataset
+        fg = folium.FeatureGroup(name=dataset_title)
+        
+        for feature in features[:100]:  # Limit features per dataset
+            geom = feature.get('geometry', {})
+            props = feature.get('properties', {})
+            geom_type = geom.get('type', '')
+            
+            # Build popup content
+            popup_lines = [f"<b>{dataset_title}</b>"]
+            for key, value in list(props.items())[:5]:  # Show first 5 properties
+                if value and str(value).strip():
+                    popup_lines.append(f"{key}: {value}")
+            popup_html = "<br>".join(popup_lines)
+            
+            try:
+                if geom_type == 'Point':
+                    coords = geom.get('coordinates', [])
+                    if len(coords) >= 2:
+                        folium.CircleMarker(
+                            location=[coords[1], coords[0]],
+                            radius=6,
+                            color=color,
+                            fill=True,
+                            fillColor=color,
+                            fillOpacity=0.7,
+                            popup=popup_html
+                        ).add_to(fg)
+                        
+                elif geom_type in ['Polygon', 'MultiPolygon']:
+                    folium.GeoJson(
+                        feature,
+                        style_function=lambda x, c=color: {
+                            'fillColor': c,
+                            'color': c,
+                            'weight': 2,
+                            'fillOpacity': 0.4
+                        },
+                        popup=folium.Popup(popup_html, max_width=300)
+                    ).add_to(fg)
+                    
+                elif geom_type in ['LineString', 'MultiLineString']:
+                    folium.GeoJson(
+                        feature,
+                        style_function=lambda x, c=color: {
+                            'color': c,
+                            'weight': 3
+                        },
+                        popup=folium.Popup(popup_html, max_width=300)
+                    ).add_to(fg)
+            except Exception as e:
+                logger.debug(f"Error adding feature: {e}")
+                continue
+        
+        fg.add_to(m)
+    
+    # Add layer control if multiple datasets
+    if len(feature_data_list) > 1:
+        folium.LayerControl().add_to(m)
+    
+    return m
+
+
+def create_conservation_chart(feature_data_list, chart_type='bar'):
+    """
+    Create a Plotly chart from conservation feature data.
+    
+    Args:
+        feature_data_list: List of dicts with 'dataset' and 'features' keys
+        chart_type: Type of chart ('bar', 'pie')
+        
+    Returns:
+        Plotly figure
+    """
+    if not feature_data_list:
+        return None
+    
+    # Aggregate data by dataset
+    chart_data = []
+    for data in feature_data_list:
+        dataset = data.get('dataset', {})
+        features = data.get('features', [])
+        chart_data.append({
+            'Dataset': dataset.get('title', 'Unknown')[:40],
+            'Feature Count': len(features),
+            'Views': dataset.get('views', 0)
+        })
+    
+    if not chart_data:
+        return None
+    
+    df = pd.DataFrame(chart_data)
+    
+    if chart_type == 'pie':
+        fig = px.pie(
+            df,
+            values='Feature Count',
+            names='Dataset',
+            title='Conservation Features by Dataset',
+            color_discrete_sequence=px.colors.qualitative.Set2
+        )
+    else:  # Default to bar
+        fig = px.bar(
+            df,
+            x='Dataset',
+            y='Feature Count',
+            color='Dataset',
+            title='Conservation Features by Dataset',
+            color_discrete_sequence=px.colors.qualitative.Set2
+        )
+        fig.update_layout(xaxis_tickangle=-45)
+    
+    fig.update_layout(
+        template='plotly_white',
+        showlegend=True,
+        height=400
+    )
+    
+    return fig
+
+
+def fetch_conservation_features_for_query(parsed_query):
+    """
+    Fetch conservation features based on parsed query.
+    
+    Returns:
+        Tuple of (feature_data_list, status_message, metadata_list)
+    """
+    if not EXTERNAL_APIS_AVAILABLE or external_data is None:
+        return [], "External APIs not available", []
+    
+    topic = parsed_query.get("conservation_topic", "forests")
+    
+    # Topic detection from query
+    query_keywords = {
+        "deforestation": "forests",
+        "forest": "forests",
+        "wildlife": "wildlife",
+        "endangered": "wildlife",
+        "species": "wildlife",
+        "marine": "oceans",
+        "ocean": "oceans",
+        "coral": "oceans",
+        "protected": "wildlife",
+        "river": "freshwater",
+        "water": "freshwater",
+        "climate": "climate",
+        "carbon": "climate"
+    }
+    
+    all_text = " ".join(parsed_query.get("locations", []) + parsed_query.get("indicators", [])).lower()
+    for keyword, topic_name in query_keywords.items():
+        if keyword in all_text:
+            topic = topic_name
+            break
+    
+    try:
+        # Fetch features from GLOBIL
+        feature_data = external_data.fetch_conservation_features(
+            topic,
+            max_datasets=3,
+            max_features=200
+        )
+        
+        if not feature_data:
+            # Fallback to metadata only
+            datasets = external_data.search_conservation_data(topic, limit=10)
+            return [], f"No feature data available for {topic}. Showing metadata only.", datasets
+        
+        total_features = sum(d.get('feature_count', 0) for d in feature_data)
+        datasets_found = len(feature_data)
+        
+        return (
+            feature_data,
+            f"Found {total_features} features from {datasets_found} datasets about {topic}",
+            [d.get('dataset', {}) for d in feature_data]
+        )
+        
+    except Exception as e:
+        logger.error(f"Error fetching conservation features: {e}")
+        return [], f"Error fetching data: {str(e)}", []
+
+
 def fetch_geospatial_data(parsed_query):
     """
     Fetch and process geospatial data based on parsed query
@@ -900,33 +1158,82 @@ def process_query(user_query, output_format, chart_type, map_style, color_scheme
         # EXTERNAL API ROUTING: Conservation Data (WWF GLOBIL)
         # =================================================================
         if data_source == "conservation":
-            datasets, status_msg = fetch_conservation_data(parsed)
-            
-            if not datasets:
-                error_msg = f"❌ **Conservation Data Query Failed**\n\n{status_msg}\n\n"
-                error_msg += "**Available Topics:**\n"
-                error_msg += "- forests, wildlife, oceans, freshwater, climate\n\n"
-                error_msg += "**Try queries like:**\n"
-                error_msg += "- 'Search for deforestation datasets'\n"
-                error_msg += "- 'Find wildlife conservation data'\n"
-                error_msg += "- 'Show ocean protection datasets'"
-                return None, None, None, error_msg, None, None
+            # Try to fetch actual feature data first
+            feature_data, status_msg, metadata = fetch_conservation_features_for_query(parsed)
             
-            # Create conservation datasets table
-            cons_df = create_conservation_table(datasets)
+            map_html = None
+            chart_fig = None
+            map_file = None
+            csv_file = None
             
-            # Save to CSV
-            csv_file = tempfile.NamedTemporaryFile(delete=False, suffix='.csv', mode='w', encoding='utf-8')
-            cons_df.to_csv(csv_file.name, index=False)
-            csv_file = csv_file.name
+            if feature_data:
+                # We have actual feature data - create map and chart
+                cons_map = create_conservation_map(feature_data, map_style)
+                if cons_map:
+                    map_html = cons_map._repr_html_()
+                    map_file = tempfile.NamedTemporaryFile(delete=False, suffix='.html', mode='w', encoding='utf-8')
+                    cons_map.save(map_file.name)
+                    map_file = map_file.name
+                
+                # Create chart showing feature counts
+                chart_fig = create_conservation_chart(feature_data, chart_type if chart_type in ['bar', 'pie'] else 'bar')
+                
+                # Create table from feature properties
+                table_rows = []
+                for data in feature_data:
+                    dataset_title = data.get('dataset', {}).get('title', 'Unknown')
+                    for feat in data.get('features', [])[:50]:  # Limit rows
+                        props = feat.get('properties', {})
+                        row = {'Dataset': dataset_title}
+                        # Get first few meaningful properties
+                        for key, val in list(props.items())[:4]:
+                            if val and str(val).strip() and key.lower() not in ['objectid', 'fid', 'shape']:
+                                row[key] = str(val)[:50]
+                        if len(row) > 1:
+                            table_rows.append(row)
+                
+                if table_rows:
+                    cons_df = pd.DataFrame(table_rows)
+                else:
+                    # Fallback to metadata table
+                    cons_df = create_conservation_table(metadata)
+                
+                summary = f"🔍 **Query:** {user_query}\n\n"
+                summary += f"📍 **Data Source:** WWF GLOBIL (ArcGIS Hub)\n"
+                summary += f"🌿 **Topic:** {parsed.get('conservation_topic', 'General')}\n"
+                summary += f"📊 **Results:** {status_msg}\n\n"
+                summary += "🗺️ *Map shows conservation areas with colored layers*\n"
+                summary += "📊 *Chart shows feature distribution across datasets*"
+            else:
+                # Fallback to metadata only (no feature data available)
+                datasets, fallback_msg = fetch_conservation_data(parsed)
+                
+                if not datasets:
+                    error_msg = f"❌ **Conservation Data Query Failed**\n\n{fallback_msg}\n\n"
+                    error_msg += "**Available Topics:**\n"
+                    error_msg += "- forests, wildlife, oceans, freshwater, climate\n\n"
+                    error_msg += "**Try queries like:**\n"
+                    error_msg += "- 'Search for deforestation datasets'\n"
+                    error_msg += "- 'Find wildlife conservation data'\n"
+                    error_msg += "- 'Show ocean protection datasets'"
+                    return None, None, None, error_msg, None, None
+                
+                cons_df = create_conservation_table(datasets)
+                
+                summary = f"🔍 **Query:** {user_query}\n\n"
+                summary += f"📍 **Data Source:** WWF GLOBIL (ArcGIS Hub)\n"
+                summary += f"🌿 **Topic:** {parsed.get('conservation_topic', 'General')}\n"
+                summary += f"📊 **Results:** {status_msg or fallback_msg}\n\n"
+                summary += "ℹ️ *Feature geometries not available for these datasets*\n"
+                summary += "💡 *Showing dataset metadata table*"
             
-            summary = f"🔍 **Query:** {user_query}\n\n"
-            summary += f"📍 **Data Source:** WWF GLOBIL (ArcGIS Hub)\n"
-            summary += f"🌿 **Topic:** {parsed.get('conservation_topic', 'General')}\n"
-            summary += f"📊 **Results:** {status_msg}\n\n"
-            summary += "💡 *Conservation datasets from World Wildlife Fund*"
+            # Save table to CSV
+            if cons_df is not None and not cons_df.empty:
+                csv_file = tempfile.NamedTemporaryFile(delete=False, suffix='.csv', mode='w', encoding='utf-8')
+                cons_df.to_csv(csv_file.name, index=False)
+                csv_file = csv_file.name
             
-            return None, None, cons_df, summary, None, csv_file
+            return map_html, chart_fig, cons_df, summary, map_file, csv_file
         
         # =================================================================
         # DEFAULT: Country/Region Data (Natural Earth)

data_utils.py

@@ -445,7 +445,55 @@ class ExternalDataHandler:
             return []
         
         return self._globil.get_public_feature_layers(category)
-    
+
+    def fetch_conservation_features(
+        self,
+        topic: str,
+        max_datasets: int = 3,
+        max_features: int = 500
+    ) -> List[Dict]:
+        """
+        Fetch actual feature data from conservation datasets.
+        
+        Args:
+            topic: Conservation topic to search
+            max_datasets: Maximum datasets to query
+            max_features: Maximum features per dataset
+            
+        Returns:
+            List of dictionaries with dataset info and GeoJSON features
+        """
+        self._ensure_initialized()
+        if not self._globil:
+            return []
+        
+        return self._globil.search_and_fetch_features(
+            topic,
+            max_datasets=max_datasets,
+            max_features_per_dataset=max_features
+        )
+
+    def get_conservation_dataset_features(
+        self,
+        item_id: str,
+        max_features: int = 1000
+    ) -> Optional[Dict]:
+        """
+        Get features from a specific conservation dataset.
+        
+        Args:
+            item_id: ArcGIS item ID
+            max_features: Maximum features to return
+            
+        Returns:
+            Dictionary with dataset info and GeoJSON features
+        """
+        self._ensure_initialized()
+        if not self._globil:
+            return None
+        
+        return self._globil.get_dataset_with_features(item_id, max_features)
+
     # =========================================================================
     # Utility Methods
     # =========================================================================

external_apis.py

@@ -474,6 +474,161 @@ class GlobILClient:
         
         return feature_layers
 
+    def get_feature_service_url(self, item_id: str) -> Optional[str]:
+        """
+        Get the Feature Service URL for an ArcGIS item.
+        
+        Args:
+            item_id: ArcGIS item ID
+            
+        Returns:
+            Feature Service URL or None
+        """
+        info = self.get_dataset_info(item_id)
+        if info and info.get("url"):
+            return info.get("url")
+        return None
+
+    def fetch_features(
+        self,
+        service_url: str,
+        layer_index: int = 0,
+        max_features: int = 1000,
+        where: str = "1=1",
+        out_fields: str = "*",
+        return_geometry: bool = True
+    ) -> Dict:
+        """
+        Fetch features from an ArcGIS Feature Service.
+        
+        Args:
+            service_url: Base URL of the Feature Service
+            layer_index: Layer index (usually 0)
+            max_features: Maximum features to return
+            where: SQL where clause filter
+            out_fields: Fields to return (* for all)
+            return_geometry: Whether to include geometries
+            
+        Returns:
+            GeoJSON-like dictionary with features
+        """
+        # Construct query URL
+        query_url = f"{service_url}/{layer_index}/query"
+        
+        params = {
+            "where": where,
+            "outFields": out_fields,
+            "returnGeometry": str(return_geometry).lower(),
+            "f": "geojson",  # Request GeoJSON format
+            "resultRecordCount": max_features
+        }
+        
+        try:
+            response = self.session.get(
+                query_url,
+                params=params,
+                timeout=API_CONFIG.DEFAULT_TIMEOUT * 2  # Longer timeout for feature data
+            )
+            response.raise_for_status()
+            data = response.json()
+            
+            # Handle ArcGIS error responses
+            if "error" in data:
+                logger.warning(f"ArcGIS error: {data['error'].get('message', 'Unknown error')}")
+                return {"type": "FeatureCollection", "features": []}
+            
+            return data
+            
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Feature fetch error: {e}")
+            return {"type": "FeatureCollection", "features": []}
+
+    def search_and_fetch_features(
+        self,
+        topic: str,
+        max_datasets: int = 3,
+        max_features_per_dataset: int = 500
+    ) -> List[Dict]:
+        """
+        Search for datasets and fetch actual feature data.
+        
+        Args:
+            topic: Conservation topic to search
+            max_datasets: Maximum datasets to query
+            max_features_per_dataset: Features per dataset
+            
+        Returns:
+            List of dictionaries with dataset info and features
+        """
+        results = []
+        
+        # Get feature layers for topic
+        layers = self.get_public_feature_layers(topic)[:max_datasets]
+        
+        for layer in layers:
+            item_id = layer.get("id")
+            if not item_id:
+                continue
+            
+            # Get feature service URL
+            service_url = self.get_feature_service_url(item_id)
+            if not service_url:
+                continue
+            
+            # Fetch features
+            geojson = self.fetch_features(
+                service_url,
+                max_features=max_features_per_dataset
+            )
+            
+            features = geojson.get("features", [])
+            if features:
+                results.append({
+                    "dataset": layer,
+                    "features": features,
+                    "feature_count": len(features),
+                    "service_url": service_url
+                })
+        
+        return results
+
+    def get_dataset_with_features(
+        self,
+        item_id: str,
+        max_features: int = 1000
+    ) -> Optional[Dict]:
+        """
+        Get a specific dataset with its features.
+        
+        Args:
+            item_id: ArcGIS item ID
+            max_features: Maximum features to return
+            
+        Returns:
+            Dictionary with dataset info and features
+        """
+        info = self.get_dataset_info(item_id)
+        if not info:
+            return None
+        
+        service_url = info.get("url")
+        if not service_url:
+            return None
+        
+        geojson = self.fetch_features(service_url, max_features=max_features)
+        
+        return {
+            "dataset": {
+                "id": item_id,
+                "title": info.get("title"),
+                "snippet": info.get("snippet"),
+                "type": info.get("type"),
+                "url": service_url
+            },
+            "geojson": geojson,
+            "feature_count": len(geojson.get("features", []))
+        }
+
 
 # Conservation data topics available in GLOBIL
 GLOBIL_TOPICS = {