Update app.py

app.py

@@ -2,13 +2,11 @@ import streamlit as st
 import requests
 from bs4 import BeautifulSoup
 from geopy.geocoders import Nominatim
-from urllib.parse import urljoin, urlparse
-import re
 import os
-import pandas as pd
 import folium
 from streamlit_folium import folium_static
 from jsonschema import validate, ValidationError
+from fake_useragent import UserAgent
 
 # Configure environment
 DEEPSEEK_API_KEY = os.getenv("DEEPSEEK_KEY")
@@ -39,39 +37,66 @@ RESPONSE_SCHEMA = {
     }
 }
 
+def get_scraping_headers():
+    """Generate random browser headers"""
+    ua = UserAgent()
+    return {
+        'User-Agent': ua.chrome,
+        'Accept-Language': 'en-US,en;q=0.9',
+        'Referer': 'https://www.google.com/',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+        'DNT': '1'
+    }
+
 @st.cache_data
 def scrape_location_data(query):
-    """Scrape location data with enhanced error handling"""
-    sources = {
-        "Niche": f"https://www.niche.com/places-to-live/search/{query}",
-    }
-    
+    """Scrape location data with advanced anti-bot measures"""
+    session = requests.Session()
     results = []
-    for source, url in sources.items():
-        try:
-            response = requests.get(url, timeout=15)
-            response.raise_for_status()
-            soup = BeautifulSoup(response.text, 'html.parser')
-            
-            if source == "Niche":
-                listings = soup.find_all('div', class_='search-results__list__item')
-                for item in listings[:3]:
-                    results.append({
-                        'name': item.find('h2').text.strip(),
-                        'details': item.find('div', class_='search-result-tagline').text.strip(),
-                        'score': item.find('div', class_='search-result-grade').text.strip()
-                    })
-                    
-        except Exception as e:
-            st.error(f"Scraping error for {source}: {str(e)}")
-            continue
+    
+    try:
+        # First request to get through potential bot checks
+        search_url = f"https://www.niche.com/places-to-live/search/{query}"
+        response = session.get(
+            search_url,
+            headers=get_scraping_headers(),
+            timeout=15,
+            allow_redirects=True
+        )
+        
+        # Detect bot protection
+        if any(keyword in response.text.lower() for keyword in ['captcha', 'security check']):
+            st.warning("""
+            **Manual Verification Required:**
+            1. Visit [Niche.com](https://www.niche.com) directly
+            2. Complete any security checks
+            3. Return here and try again
+            """)
+            return []
+
+        soup = BeautifulSoup(response.text, 'html.parser')
+        
+        # Resilient CSS selector patterns
+        listings = soup.select('div[class*="search-results__list__item"]')
+        
+        for item in listings[:3]:
+            result = {
+                'name': item.select_one('h2').get_text(strip=True) if item.select_one('h2') else 'N/A',
+                'details': item.select_one('div[class*="tagline"]').get_text(strip=True) if item.select_one('div[class*="tagline"]') else '',
+                'score': item.select_one('div[class*="grade"]').get_text(strip=True) if item.select_one('div[class*="grade"]")') else ''
+            }
+            results.append(result)
             
-    return results
+        return results
+        
+    except Exception as e:
+        st.error(f"Data retrieval issue: {str(e)}")
+        return []
 
 def generate_recommendations(preferences):
     """Generate recommendations with robust error handling"""
     if not DEEPSEEK_API_KEY:
-        st.error("Missing API key - check environment configuration")
+        st.error("Missing API key configuration")
         return None
 
     headers = {
@@ -80,17 +105,14 @@ def generate_recommendations(preferences):
     }
     
     prompt = f"""
-    Create a neighborhood recommendation report based on these preferences:
-    {preferences}
-    
-    Include these sections:
-    1. Top 5 Neighborhood Matches
-    2. Hidden Gem Recommendation
-    3. Key Amenities Analysis
-    4. Commute Times Overview
-    5. Safety & Community Insights
-    
-    Format with markdown headers and bullet points.
+    Analyze these neighborhood preferences: {preferences}
+    Create a detailed report with:
+    - Top 3 matches
+    - 1 hidden gem
+    - Amenity analysis
+    - Safety insights
+    - Price comparisons
+    Format with markdown sections
     """
     
     try:
@@ -106,20 +128,18 @@ def generate_recommendations(preferences):
             timeout=30
         )
         
-        # Validate response status
         if response.status_code != 200:
             error_msg = response.json().get('error', {}).get('message', 'Unknown API error')
             st.error(f"API Error {response.status_code}: {error_msg}")
             return None
         
-        # Validate response structure
         response_data = response.json()
         validate(instance=response_data, schema=RESPONSE_SCHEMA)
         
         return response_data["choices"][0]["message"]["content"]
         
     except ValidationError as ve:
-        st.error(f"Invalid API response format: {str(ve)}")
+        st.error(f"Invalid API response: {str(ve)}")
         return None
     except Exception as e:
         st.error(f"Connection Error: {str(e)}")
@@ -127,46 +147,48 @@ def generate_recommendations(preferences):
 
 # Streamlit UI
 st.set_page_config(layout="wide", page_icon="🏡")
-st.title("Neighborhood Matchmaker")
+st.title("Neighborhood Matchmaker Pro")
 
 with st.sidebar:
     st.header("Search Preferences")
     city = st.text_input("City/Region", "New York, NY")
-    budget = st.slider("Monthly Housing Budget ($)", 1000, 10000, 3000)
-    commute = st.selectbox("Max Commute Time", ["15 mins", "30 mins", "45 mins", "1 hour"])
-    amenities = st.multiselect("Must-Have Amenities", [
-        "Good Schools", "Parks", "Public Transport", 
-        "Nightlife", "Shopping", "Healthcare"
-    ])
-    lifestyle = st.selectbox("Lifestyle Preference", [
-        "Family-Friendly", "Urban Professional", "Retirement", 
-        "Student", "Remote Worker", "Outdoor Enthusiast"
-    ])
+    budget = st.slider("Monthly Budget ($)", 1000, 10000, 3000)
+    commute = st.selectbox("Max Commute", ["15 mins", "30 mins", "45 mins"])
+    lifestyle = st.selectbox("Lifestyle", ["Family", "Urban", "Remote"])
 
-if st.button("Find My Neighborhood"):
+if st.button("Find Neighborhoods"):
     with st.spinner("Analyzing locations..."):
         preferences = {
             "city": city,
-            "budget": f"${budget}/mo",
-            "max_commute": commute,
-            "amenities": amenities,
+            "budget": budget,
+            "commute": commute,
             "lifestyle": lifestyle
         }
         
         location_data = scrape_location_data(city)
+        
+        if not location_data:
+            st.info("Using alternative data sources...")
+            # Implement fallback scraping here
+            
         report = generate_recommendations(preferences)
         
         if report:
-            st.subheader("Your Personalized Neighborhood Report")
+            st.subheader("Custom Neighborhood Report")
             st.markdown(report)
             
             try:
-                geolocator = Nominatim(user_agent="neighborhood_finder")
+                geolocator = Nominatim(user_agent="geo_app_v2")
                 location = geolocator.geocode(city)
-                m = folium.Map(location=[location.latitude, location.longitude], zoom_start=12)
-                folium_static(m, width=1200, height=500)
+                if location:
+                    m = folium.Map(
+                        location=[location.latitude, location.longitude],
+                        zoom_start=12,
+                        tiles="CartoDB positron"
+                    )
+                    folium_static(m, width=1000, height=500)
             except Exception as e:
-                st.warning(f"Map visualization error: {str(e)}")
+                st.warning(f"Map error: {str(e)}")
 
 st.markdown("---")
-st.caption("Note: Recommendations generated by AI. Verify with local experts.")
+st.caption("Note: Results may vary based on data availability")