Update app.py

app.py

@@ -20,6 +20,55 @@ try:
 except:
     API_KEY = None
 
+# ==========================================
+# 1. UNIVERSAL FILTER LISTS (THE FIX)
+# ==========================================
+# Filters out Schools, Doctors, Industrial services, etc.
+UNIVERSAL_BAD_TERMS = [
+    "dr.", "dds", "md", "phd", "lcsw", "medical", "clinic", "health", "rehab", 
+    "therapy", "counseling", "chiropractor", "dental", "orthodontics", "hospital", 
+    "ambulance", "transport", "emergency", "veterinary", "vision center",
+    "school", "university", "college", "academy", "campus", "library", "learning", 
+    "student", "alum", "education", "institute", "dorm", "residence",
+    "atm", "kiosk", "redbox", "coinme", "fuel", "gas", "repair", "service", 
+    "collision", "towing", "plumbing", "hvac", "electric", "tree", "lawn", 
+    "gutter", "cleaning", "storage", "warehouse", "distribution", "mural", "statue",
+    "part", "accessories"
+]
+
+# Filters out departments inside Big Box stores
+DEPARTMENT_TERMS = [
+    "grocery", "deli", "bakery", "pharmacy", "optical", "hearing", "photo", 
+    "portrait", "garden", "nursery", "mobile", "tech", "geek", "pickup", 
+    "money", "bank", "cafe", "bistro", "snack", "food court", "customer service"
+]
+
+BRAND_FLOORS = {
+  "Macy's": 2, "JCPenney": 2, "Nordstrom": 2, "Sears": 2, "IKEA": 2,
+  "Target": 1, "Walmart": 1, "Costco": 1, "Home Depot": 1, "Lowe's": 1,
+  "Barnes & Noble": 1, "Dick's Sporting Goods": 1, "Kohl's": 1
+}
+
+BRAND_AVG_AREA = {
+    "IKEA": 28000, "Walmart": 15000, "Costco": 14000, "Sam's Club": 13000, 
+    "Meijer": 18000, "Target": 12000, "Home Depot": 10000, "Lowe's": 10000, 
+    "Kroger": 6000, "Safeway": 5000, "Whole Foods": 4000, "Macy's": 16000, 
+    "JCPenney": 10000, "Sears": 12000, "Kohl's": 8000, "Dick's Sporting Goods": 4500,
+    "T.J. Maxx": 2800, "Marshalls": 2800, "Ross Dress for Less": 2800, 
+    "Old Navy": 1400, "Barnes & Noble": 2500, "Best Buy": 3500, "Staples": 2000, 
+    "Office Depot": 2000, "PetSmart": 1800, "Petco": 1400, "Trader Joe's": 1200, 
+    "Aldi": 1500, "Lidl": 1500, "Ace Hardware": 800, "DSW Designer Shoe Warehouse": 2000
+}
+
+SEARCH_LIST = [
+    "Walmart", "Target", "Kmart", "Sears", "Kohl's", "Macy's", "JCPenney", 
+    "TJX", "T.J. Maxx", "Marshalls", "HomeGoods", "Ross Dress for Less", "Burlington", 
+    "Dick's Sporting Goods", "Albertsons", "Safeway", "Home Depot", "Lowe's", "Best Buy", 
+    "IKEA", "Bob's Discount Furniture", "Barnes & Noble", "Office Depot", "OfficeMax", 
+    "Staples", "PetSmart", "Petco", "Kroger", "Meijer", "Costco", "Sam's Club", 
+    "Whole Foods", "Trader Joe's", "Michaels", "Aldi", "Old Navy", "Ace Hardware"
+]
+
 # ==========================================
 # HELPER FUNCTIONS
 # ==========================================
@@ -43,8 +92,6 @@ def load_geodata_to_polygon(file_obj):
         if target_kml:
             gdf = gpd.read_file(target_kml)
             
-            # --- UNIVERSAL FIX: FORCE 2D ---
-            # This ensures ANY KML with height data works correctly
             def force_2d(geometry):
                 if geometry.has_z:
                     return transform(lambda x, y, z=None: (x, y), geometry)
@@ -58,12 +105,7 @@ def load_geodata_to_polygon(file_obj):
 
 def get_roof_area(lat, lng, api_key):
     base_url = "https://solar.googleapis.com/v1/buildingInsights:findClosest"
-    params = {
-        "location.latitude": lat,
-        "location.longitude": lng,
-        "requiredQuality": "HIGH",
-        "key": api_key
-    }
+    params = {"location.latitude": lat, "location.longitude": lng, "requiredQuality": "HIGH", "key": api_key}
     try:
         resp = requests.get(base_url, params=params)
         data = resp.json()
@@ -95,45 +137,14 @@ def get_osm_physics(lat, lng):
         pass
     return None, None
 
-# DATA CONSTANTS
-BRAND_FLOORS = {
-  "Macy's": 2, "JCPenney": 2, "Nordstrom": 2, "Sears": 2, "IKEA": 2,
-  "Target": 1, "Walmart": 1, "Costco": 1, "Home Depot": 1, "Lowe's": 1,
-  "Barnes & Noble": 1, "Dick's Sporting Goods": 1, "Kohl's": 1
-}
-
-BRAND_AVG_AREA = {
-    "IKEA": 28000, "Walmart": 15000, "Costco": 14000, "Sam's Club": 13000, 
-    "Meijer": 18000, "Target": 12000, "Home Depot": 10000, "Lowe's": 10000, 
-    "Kroger": 6000, "Safeway": 5000, "Whole Foods": 4000, "Macy's": 16000, 
-    "JCPenney": 10000, "Sears": 12000, "Kohl's": 8000, "Dick's Sporting Goods": 4500,
-    "T.J. Maxx": 2800, "Marshalls": 2800, "Ross Dress for Less": 2800, 
-    "Old Navy": 1400, "Barnes & Noble": 2500, "Best Buy": 3500, "Staples": 2000, 
-    "Office Depot": 2000, "PetSmart": 1800, "Petco": 1400, "Trader Joe's": 1200, 
-    "Aldi": 1500, "Lidl": 1500, "Ace Hardware": 800, "DSW Designer Shoe Warehouse": 2000
-}
-
-SEARCH_LIST = [
-    "Walmart", "Target", "Kmart", "Sears", "Kohl's", "Macy's", "JCPenney", 
-    "TJX", "TJX Companies", "T.J. Maxx", "Marshalls", "HomeGoods", "HomeSense", 
-    "Ross", "Ross Dress for Less", "Burlington", "Dick's Sporting Goods", 
-    "Albertsons", "Safeway", "Home Depot", "Lowe's", "Best Buy", 
-    "IKEA", "Bob's Furniture", "Bob's Discount Furniture", "Raymour & Flanigan", 
-    "Barnes & Noble", "Office Depot", "OfficeMax", "Staples", "Lowe",
-    "PetSmart", "Petco", "Kroger", "Meijer", "Costco", "BJ's Wholesale Club", 
-    "Sam's Club", "Whole Foods", "ShopRite", "Stop & Shop", "Trader Joe's", 
-    "Michaels", "Lidl", "Aldi", "DSW Designer Shoe Warehouse", "Old Navy", 
-    "Ace", "Ace Hardware", "Hobby Lobby", "Trader Joes"
-]
-
 # ==========================================
-# MAIN LOGIC WITH GENERATOR (YIELD)
+# MAIN LOGIC
 # ==========================================
 def process_data(file_obj):
     if not API_KEY:
         yield "❌ API Key not found! Set GOOGLE_API_KEY in Secrets.", None
         return
-    
+        
     if file_obj is None:
         yield "❌ Please upload a file.", None
         return
@@ -145,31 +156,24 @@ def process_data(file_obj):
         yield "❌ Failed to read KML/KMZ file.", None
         return
 
-    # --- UNIVERSAL AREA LIMIT CHECK ---
-    try:
-        gs = gpd.GeoSeries([polygon], crs="EPSG:4326")
-        gs_proj = gs.to_crs(epsg=6933)
-        area_sq_meters = gs_proj.area.iloc[0]
-        limit_sq_meters = 250_000_000
-        
-        if area_sq_meters > limit_sq_meters:
-            yield f"⚠️ AREA TOO LARGE: {area_sq_meters:,.0f} sq m. (Limit: {limit_sq_meters:,.0f}). Upload a smaller file.", None
-            return 
-    except:
-        pass
+    # Check Area Limit
+    gs = gpd.GeoSeries([polygon], crs="EPSG:4326")
+    gs_proj = gs.to_crs(epsg=6933)
+    area_sq_meters = gs_proj.area.iloc[0]
+    if area_sq_meters > 250_000_000:
+        yield f"⚠️ AREA TOO LARGE: {area_sq_meters:,.0f} sq m. (Limit: 250M).", None
+        return 
 
     gmaps = googlemaps.Client(key=API_KEY)
     results = []
     seen_ids = set()
     total_brands = len(SEARCH_LIST)
     
-    # LOOP THROUGH BRANDS
+    # 1. SEARCH LOOP
     for i, brand in enumerate(SEARCH_LIST):
         yield f"🔍 Scanning Brand {i+1}/{total_brands}: {brand}...", None
         
         try:
-            # --- 1. DEEP SEARCH (Pagination enabled) ---
-            # 10km radius + 3 Pages of results ensures we don't miss local stores
             places = gmaps.places_nearby(
                 location=(polygon.centroid.y, polygon.centroid.x),
                 radius=10000, 
@@ -192,48 +196,40 @@ def process_data(file_obj):
                 if pid in seen_ids: continue
                 
                 name = p.get('name')
-                
-                # --- 2. UNIVERSAL NAME VERIFICATION ---
-                # Check: Is the brand name actually inside the store name?
-                # This prevents "Meijer" showing up when searching for "Walmart"
-                
-                # Normalize strings (remove case, apostrophes, periods)
                 name_clean = name.lower().replace("'", "").replace(".", "")
                 brand_clean = brand.lower().replace("'", "").replace(".", "")
 
+                # A. UNIVERSAL NAME CHECK
                 if brand_clean not in name_clean:
-                    # Exceptions for tricky names (TJX/Lowe)
                     if brand_clean == "tjx" and "t.j. maxx" in name_clean: pass
                     elif brand_clean == "lowe" and "lowe's" in name_clean: pass
-                    else: continue # Reject the result
-                
-                # --- 3. BAD KEYWORD FILTER ---
-                # Filters out ATMs, Doctors, Fuel, Vision Centers, etc.
-                bad_terms = ["atm", "redbox", "kiosk", "coinme", "gas", "fuel", "lcsw", "dr.", "dds", "hair", "salon", "studio", "tire", "repair"]
-                if any(term in name_clean for term in bad_terms): continue
-                
+                    else: continue
+
+                # B. UNIVERSAL BAD WORD FILTER (Strict)
+                if any(term in name_clean for term in UNIVERSAL_BAD_TERMS): continue
+
                 lat = p['geometry']['location']['lat']
                 lng = p['geometry']['location']['lng']
                 
-                # --- 4. STRICT CONTAINMENT ---
-                # Only keep if strictly inside the KML polygon
+                # C. STRICT CONTAINMENT
                 if not polygon.contains(Point(lng, lat)): continue
                 seen_ids.add(pid)
                 
-                # Get Data
+                # FETCH DATA
                 roof_area = get_roof_area(lat, lng, API_KEY)
                 height, floors = get_osm_physics(lat, lng)
                 
-                # Logic
+                # DATA FILLING
                 source_note = "SolarAPI"
                 if roof_area is None:
                     roof_area = BRAND_AVG_AREA.get(brand, 2500.0)
                     source_note = "Brand_Avg (Missing)"
                 else:
+                    # Universal Mall Logic
                     if roof_area > 30000 and brand not in ["IKEA", "Costco", "Meijer", "Sam's Club", "Walmart"]:
                         roof_area = BRAND_AVG_AREA.get(brand, 2500.0)
                         source_note = "Brand_Avg (Mall detected)"
-                    elif roof_area < 500 and brand not in ["Ace Hardware", "Trader Joe's", "GameStop"]:
+                    elif roof_area < 500 and brand not in ["Ace Hardware", "Trader Joe's"]:
                         roof_area = BRAND_AVG_AREA.get(brand, 2500.0)
                         source_note = "Brand_Avg (Too Small detected)"
                 
@@ -241,9 +237,14 @@ def process_data(file_obj):
                 if height is None: height = floors * 6.0
                 
                 results.append({
-                    'Name': name, 'Brand': brand, 'Latitude': lat, 'Longitude': lng,
-                    'Height_m': round(height, 2), 'Num_Floors': int(floors),
-                    'Area_sqm': round(roof_area, 2), 'Data_Source': source_note
+                    'Name': name, 
+                    'Brand': brand, 
+                    'Latitude': lat, 
+                    'Longitude': lng,
+                    'Height_m': round(height, 2), 
+                    'Num_Floors': int(floors),
+                    'Area_sqm': round(roof_area, 2),
+                    'Data_Source': source_note
                 })
         except:
             pass
@@ -252,30 +253,65 @@ def process_data(file_obj):
         yield "❌ No stores found in this area.", None
         return
 
+    # ==========================================
+    # 2. UNIVERSAL POST-PROCESSING (THE LOGIC UPGRADE)
+    # ==========================================
+    yield "🧹 Performing Universal Deduplication...", None
+    
     df = pd.DataFrame(results)
+
+    # A. Remove Departments (e.g. Target Grocery, Meijer Deli)
+    df = df[~df['Name'].str.contains('|'.join(DEPARTMENT_TERMS), case=False, na=False)]
+
+    # B. Spatial Deduplication (Group by Brand + Location)
+    # Creates a grid ID approx 11 meters. If multiple of same brand exist, keep the one with shortest name.
+    df['Loc_ID'] = df['Latitude'].round(4).astype(str) + "_" + df['Longitude'].round(4).astype(str)
+    
+    # Sort by Name Length (Shortest name usually "Target", longest usually "Target Grocery ...")
+    df['Name_Len'] = df['Name'].str.len()
+    df = df.sort_values(by=['Brand', 'Loc_ID', 'Name_Len'])
     
-    # --- FINAL CLEANUP (Sub-departments) ---
-    # Double check to remove things like "Walmart Vision Center"
-    bad_keywords = ['Mobile', 'Salon', 'Floral', 'Bakery', 'Pharmacy', 'Optical', 'Geek Squad', 'Photo', 'Tire', 'Vision']
-    df = df[~df['Name'].str.contains('|'.join(bad_keywords), case=False, na=False)]
+    # Keep only the first entry per brand at that specific lat/long
+    df = df.drop_duplicates(subset=['Brand', 'Loc_ID'], keep='first')
+    df = df.drop(columns=['Loc_ID', 'Name_Len'])
+
+    # C. Universal Strip Mall Splitter
+    # If different brands share the exact same roof area (down to decimal), split the area
+    df['Tenant_Count'] = df.groupby('Area_sqm')['Area_sqm'].transform('count')
+    
+    # Logic: If 3 tenants share 3000sqm, each gets 1000sqm
+    df['Final_Area_sqm'] = df.apply(
+        lambda x: x['Area_sqm'] / x['Tenant_Count'] if x['Tenant_Count'] > 1 and x['Area_sqm'] > 5000 else x['Area_sqm'], 
+        axis=1
+    )
+    
+    # Update Data Source note
+    df['Data_Source'] = df.apply(
+        lambda x: x['Data_Source'] + f" (Split w/ {x['Tenant_Count']-1} tenants)" if x['Tenant_Count'] > 1 and x['Area_sqm'] > 5000 else x['Data_Source'],
+        axis=1
+    )
+
+    # Clean Export
+    final_cols = ['Name', 'Brand', 'Latitude', 'Longitude', 'Height_m', 'Num_Floors', 'Final_Area_sqm', 'Data_Source']
+    df_final = df[final_cols].rename(columns={'Final_Area_sqm': 'Area_sqm'})
     
-    output_path = "Building_Inventory.csv"
-    df.to_csv(output_path, index=False)
+    output_path = "Universal_Building_Inventory.csv"
+    df_final.to_csv(output_path, index=False)
     
-    yield f"✅ Success! Found {len(df)} stores.", output_path
+    yield f"✅ Success! Found {len(df_final)} unique commercial assets.", output_path
 
 # ==========================================
 # GRADIO INTERFACE
 # ==========================================
 iface = gr.Interface(
     fn=process_data,
-    inputs=gr.File(label="Upload Polygon - - (Limit: 250,000,000 sq m)"),
+    inputs=gr.File(label="Upload Polygon (KML/KMZ)"),
     outputs=[
         gr.Textbox(label="Status Log"),
         gr.File(label="Download CSV")
     ],
-    title="🏙️ Commercial Building Inventory Generator (Test - Check- Nokia)",
-    description="Upload a KMZ file. The tool will scan for 50+ major Big Box brands, check Solar API, Places API, OpenStreetMap for Area + height/floors."
+    title="🌎 Universal Commercial Asset Scanner",
+    description="Upload any KML/KMZ. Scans for Big Box Retail. Applies universal filtering for medical/academic false positives and automatically splits strip-mall roof areas."
 )
 
 iface.launch()