fix: search integration, detailed boundaries, cleanup dead files

- Search now auto-sets department and switches to commune level
- Department selection preserved across level switches (was being cleared)
- Department highlight uses detailed vector tiles instead of simplified GeoJSON
- Postcode boundaries regenerated with finer tolerance (0.003→0.0005)
- Added gzip compression middleware for large files
- Removed dead files: prices_section.json, sections.pmtiles, communes.geojson
- Removed unused PMTiles script tag and library
- Cleaned up .gitignore for temp build directories

.gitattributes

@@ -1 +1,2 @@
 data/aggregated/prices_commune.json filter=lfs diff=lfs merge=lfs -text
+data/aggregated/postcodes.geojson filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,13 +1,17 @@
 # ---- Data (raw and intermediate - too large to commit) ----
 data/raw/
 data/processed/
+data/geo/
 # NOTE: data/aggregated/ is NOT ignored - those JSON files are needed by the app
-# The monolithic section file is replaced by per-department splits in sections/
+# Dead files that should not be committed:
 data/aggregated/prices_section.json
-# PMTiles replaced by government vector tiles (openmaptiles.geo.data.gouv.fr)
 data/aggregated/sections.pmtiles
 data/aggregated/communes.geojson
 
+# ---- Temp build artifacts ----
+.tmp_sections/
+.tmp_sections_enriched/
+
 # ---- Python ----
 __pycache__/
 *.py[cod]

app.py

@@ -8,6 +8,7 @@ Designed for Hugging Face Spaces Docker deployment on port 7860.
 from pathlib import Path
 
 from fastapi import FastAPI
+from fastapi.middleware.gzip import GZipMiddleware
 from fastapi.responses import FileResponse
 from fastapi.staticfiles import StaticFiles
 
@@ -16,9 +17,7 @@ DATA_DIR = ROOT / "data" / "aggregated"
 STATIC_DIR = ROOT / "static"
 
 app = FastAPI(title="French Property Prices", docs_url=None, redoc_url=None)
-
-# Serve static files (JS, CSS)
-app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
+app.add_middleware(GZipMiddleware, minimum_size=1000)
 
 
 @app.get("/")
@@ -26,18 +25,9 @@ async def index():
     return FileResponse(STATIC_DIR / "index.html")
 
 
-MEDIA_TYPES = {
-    ".json": "application/json",
-    ".geojson": "application/geo+json",
-    ".pmtiles": "application/octet-stream",
-}
-
+# Serve static files (JS, CSS)
+app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
 
-@app.get("/data/{filename:path}")
-async def get_data(filename: str):
-    """Serve pre-computed data files (JSON, GeoJSON, PMTiles)."""
-    path = DATA_DIR / filename
-    if not path.exists() or not path.is_file():
-        return {"error": "not found"}, 404
-    media = MEDIA_TYPES.get(path.suffix, "application/octet-stream")
-    return FileResponse(path, media_type=media)
+# Serve data files (JSON, GeoJSON, PMTiles) - StaticFiles handles range requests
+# which is required for PMTiles (HTTP range-based tile fetching)
+app.mount("/data", StaticFiles(directory=DATA_DIR), name="data")

scripts/build_postcode_geojson.py

@@ -43,15 +43,21 @@ ARRONDISSEMENT_CITIES = {
 }
 
 
-def fetch_url(url: str) -> dict | None:
-    """Fetch JSON from a URL."""
-    try:
-        req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
-        with urllib.request.urlopen(req, timeout=30) as resp:
-            return json.loads(resp.read().decode())
-    except Exception as e:
-        print(f"  WARNING: Failed to fetch {url}: {e}")
-        return None
+def fetch_url(url: str, retries: int = 3) -> dict | None:
+    """Fetch JSON from a URL with retries."""
+    for attempt in range(retries):
+        try:
+            req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
+            with urllib.request.urlopen(req, timeout=60) as resp:
+                return json.loads(resp.read().decode())
+        except Exception as e:
+            if attempt < retries - 1:
+                wait = 2 ** attempt
+                print(f"  retry {attempt+1} in {wait}s...", end=" ", flush=True)
+                time.sleep(wait)
+            else:
+                print(f"  WARNING: Failed to fetch {url}: {e}")
+                return None
 
 
 def fetch_dept_communes(dept_code: str) -> list[dict]:
@@ -128,8 +134,7 @@ def build_postcode_geojson():
             total_communes += 1
 
         # Be polite to the API
-        if i % 10 == 9:
-            time.sleep(0.5)
+        time.sleep(0.3)
 
     # Step 1b: Fetch arrondissements for Paris/Lyon/Marseille
     for city_code, city_name in ARRONDISSEMENT_CITIES.items():
@@ -169,9 +174,9 @@ def build_postcode_geojson():
     dissolved = gdf.dissolve(by="codePostal", aggfunc="first").reset_index()
     print(f"  Dissolved features: {len(dissolved)}")
 
-    # Step 4: Simplify geometries for web (tolerance ~330m, fine for zoom 11-13)
+    # Step 4: Simplify geometries for web (tolerance ~55m, smooth at zoom 12-13)
     print("\nSimplifying geometries...")
-    dissolved["geometry"] = dissolved["geometry"].simplify(tolerance=0.003, preserve_topology=True)
+    dissolved["geometry"] = dissolved["geometry"].simplify(tolerance=0.0005, preserve_topology=True)
 
     # Step 5: Export
     print(f"\nExporting to {OUTPUT_PATH}...")
@@ -184,7 +189,7 @@ def build_postcode_geojson():
 
     def round_coords(coords):
         if isinstance(coords[0], (int, float)):
-            return [round(c, 4) for c in coords]
+            return [round(c, 5) for c in coords]
         return [round_coords(c) for c in coords]
 
     for feature in geojson["features"]:

scripts/build_sections_pmtiles.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Download all per-department cadastral section GeoJSON, enrich with price data, and build PMTiles
+# Download all per-department cadastral section GeoJSON, enrich with price buckets, and build optimized PMTiles
 set -e
 
 BASEDIR="$(cd "$(dirname "$0")/.." && pwd)"
@@ -31,29 +31,37 @@ for dept in $DEPTS; do
 done
 
 echo ""
-echo "=== Step 2: Enriching GeoJSON with price data ==="
-python3 "$BASEDIR/scripts/enrich_sections.py"
+echo "=== Step 2: Enriching GeoJSON with price bucket data ==="
+uv run python "$BASEDIR/scripts/enrich_sections.py"
 
 echo ""
-echo "=== Step 3: Building PMTiles with tippecanoe ==="
+echo "=== Step 3: Building optimized PMTiles with tippecanoe ==="
 echo "Input files: $(ls "$ENRICHED"/*.geojson 2>/dev/null | wc -l)"
 
 # Use main disk for tippecanoe temp files (not /tmp which may be tmpfs)
 export TMPDIR="$TMPDIR"
 
+# Key optimizations vs the old 867MB build:
+#   -y: only include bucket + price properties (strips cadastre metadata)
+#   --coalesce-densest-as-needed: merge dense areas at low zoom (vs drop-densest which removes them)
+#   --detect-shared-borders: consistent simplification at polygon edges
+#   --simplification=10: aggressive simplification at low zoom
+#   --no-tiny-polygon-reduction: keep all polygons for choropleth coverage
 tippecanoe \
     -o "$OUTDIR/sections.pmtiles" \
-    -Z0 -z14 \
+    -Z4 -z14 \
     -l sections \
+    -y id -y bt -y ba -y bm -y pt -y pa -y pm \
+    --coalesce-densest-as-needed \
+    --detect-shared-borders \
+    --simplification=10 \
+    --no-tiny-polygon-reduction \
     --no-feature-limit \
-    --no-tile-size-limit \
-    --drop-densest-as-needed \
-    --extend-zooms-if-still-dropping \
     --force \
     "$ENRICHED"/*.geojson
 
 echo ""
 ls -lh "$OUTDIR/sections.pmtiles"
-echo "=== Done! Cleaning up ==="
+echo "=== Done! Cleaning up temp files ==="
 
-rm -rf "$TMPDIR" "$ENRICHED"
+rm -rf "$BASEDIR/.tmp_sections" "$BASEDIR/.tmp_sections_enriched"

scripts/enrich_sections.py

@@ -1,17 +1,15 @@
 """
-Enrich section GeoJSON files with pre-computed price data.
+Enrich section GeoJSON files with pre-computed price color buckets.
 
 For each department:
   1. Load the section GeoJSON (geometry)
   2. Load the section price JSON (prices)
-  3. Join prices into feature properties with short keys
-  4. Write enriched GeoJSON
-
-Short property keys to minimize tile size:
-  pt/pa/pm = wtm price (tous/appt/maison)
-  vt/va/vm = volume
-  mt/ma/mm = median
-  ct/ca/cm = confidence
+  3. Assign color buckets based on GLOBAL decile boundaries
+  4. Write enriched GeoJSON with minimal properties
+
+Properties added (short keys to minimize tile size):
+  bt/ba/bm = color bucket 1-10 (tous/appt/maison), 0 = no data
+  pt/pa/pm = raw wtm price rounded to int (for tooltip/filtering)
 """
 
 import json
@@ -23,9 +21,56 @@ PRICE_DIR = BASEDIR / "data" / "aggregated" / "sections"
 GEO_DIR = BASEDIR / ".tmp_sections"
 OUT_DIR = BASEDIR / ".tmp_sections_enriched"
 
+NUM_BUCKETS = 10
+TYPES = [("tous", "t"), ("appartement", "a"), ("maison", "m")]
+
+
+def compute_decile_boundaries(prices: list[float]) -> list[float]:
+    """Compute NUM_BUCKETS-1 boundaries for decile-based bucketing."""
+    if not prices:
+        return []
+    prices = sorted(prices)
+    n = len(prices)
+    return [prices[min(int(n * i / NUM_BUCKETS), n - 1)] for i in range(1, NUM_BUCKETS)]
+
 
-def enrich_department(dept_code: str) -> int:
-    """Enrich one department's section GeoJSON with price data. Returns feature count."""
+def assign_bucket(price: float, boundaries: list[float]) -> int:
+    """Assign a price to bucket 1-NUM_BUCKETS. Returns 0 if no data."""
+    if not boundaries or price <= 0:
+        return 0
+    for i, b in enumerate(boundaries):
+        if price <= b:
+            return i + 1
+    return NUM_BUCKETS
+
+
+def compute_global_boundaries() -> dict[str, list[float]]:
+    """Read all department price files and compute global decile boundaries."""
+    print("Computing global decile boundaries from all departments...")
+    all_prices = {suffix: [] for _, suffix in TYPES}
+
+    price_files = sorted(PRICE_DIR.glob("*.json"))
+    for pf in price_files:
+        with open(pf) as f:
+            data = json.load(f)
+        for section_id, section_data in data.items():
+            for type_key, suffix in TYPES:
+                stats = section_data.get(type_key)
+                if stats and stats.get("wtm", 0) > 0:
+                    all_prices[suffix].append(stats["wtm"])
+
+    boundaries = {}
+    for type_key, suffix in TYPES:
+        bounds = compute_decile_boundaries(all_prices[suffix])
+        boundaries[suffix] = bounds
+        print(f"  {type_key}: {len(all_prices[suffix])} values, "
+              f"deciles: {[round(b) for b in bounds]}")
+
+    return boundaries
+
+
+def enrich_department(dept_code: str, boundaries: dict) -> int:
+    """Enrich one department's section GeoJSON with bucket data."""
     geo_path = GEO_DIR / f"{dept_code}.geojson"
     price_path = PRICE_DIR / f"{dept_code}.json"
     out_path = OUT_DIR / f"{dept_code}.geojson"
@@ -47,19 +92,28 @@ def enrich_department(dept_code: str) -> int:
         props = feature.get("properties", {})
         section_id = props.get("id", "")
 
+        # Keep only the id property, strip everything else
+        new_props = {"id": section_id}
+
         if section_id in prices:
             p = prices[section_id]
             matched += 1
 
-            for suffix, type_key in [("t", "tous"), ("a", "appartement"), ("m", "maison")]:
+            for type_key, suffix in TYPES:
                 stats = p.get(type_key)
-                if stats:
-                    props[f"p{suffix}"] = round(stats.get("wtm", 0), 1)
-                    props[f"v{suffix}"] = stats.get("volume", 0)
-                    props[f"m{suffix}"] = round(stats.get("median", 0), 1)
-                    props[f"c{suffix}"] = round(stats.get("confidence", 0), 3)
-
-        feature["properties"] = props
+                if stats and stats.get("wtm", 0) > 0:
+                    wtm = stats["wtm"]
+                    new_props[f"b{suffix}"] = assign_bucket(wtm, boundaries[suffix])
+                    new_props[f"p{suffix}"] = round(wtm)
+                else:
+                    new_props[f"b{suffix}"] = 0
+                    new_props[f"p{suffix}"] = 0
+        else:
+            for _, suffix in TYPES:
+                new_props[f"b{suffix}"] = 0
+                new_props[f"p{suffix}"] = 0
+
+        feature["properties"] = new_props
 
     with open(out_path, "w") as f:
         json.dump(geo, f, separators=(",", ":"))
@@ -72,16 +126,19 @@ def enrich_department(dept_code: str) -> int:
 def main():
     OUT_DIR.mkdir(parents=True, exist_ok=True)
 
+    # Compute global boundaries first
+    boundaries = compute_global_boundaries()
+
     # Get all department codes from GeoJSON files
     dept_codes = sorted(p.stem for p in GEO_DIR.glob("*.geojson"))
     if not dept_codes:
         print(f"No GeoJSON files found in {GEO_DIR}")
         sys.exit(1)
 
-    print(f"Enriching {len(dept_codes)} departments...")
+    print(f"\nEnriching {len(dept_codes)} departments...")
     total = 0
     for code in dept_codes:
-        total += enrich_department(code)
+        total += enrich_department(code, boundaries)
 
     print(f"\nDone: {total} total features enriched")
     print(f"Output: {OUT_DIR}")

static/app.js

@@ -170,6 +170,7 @@ let searchDropdownIdx = -1;       // keyboard nav index for search dropdown
 let activeLevel = 0;              // index into LEVELS — controlled by radio buttons only
 let selectedAreaCode = null;      // code of area selected in the area list (country/region/dept)
 let selectedDeptCode = null;      // department selected for commune/postcode/section levels
+let searchedCommuneCode = null;   // commune code from search (for highlighting)
 let map;
 
 // ---- Color Interpolation ----
@@ -459,6 +460,7 @@ async function onMapLoad() {
             // Advance to next level
             activeLevel = levelIdx + 1;
             selectedAreaCode = null;
+            searchedCommuneCode = null;
             document.querySelectorAll('input[name="map-level"]')[activeLevel].checked = true;
             updateLayerVisibility();
             updateHighlight();
@@ -490,6 +492,22 @@ async function onMapLoad() {
         paint: { "fill-opacity": 0 },
     });
 
+    // Detailed department boundary from vector tiles (used at commune/postcode/section levels)
+    // This replaces the simplified local GeoJSON boundary for dept-required levels
+    map.addLayer({
+        id: "dept-vector-highlight",
+        type: "line",
+        source: "admin-tiles",
+        "source-layer": "departements",
+        layout: { visibility: "none" },
+        filter: ["==", ["get", "code"], ""],
+        paint: {
+            "line-color": document.body.classList.contains("dark-mode") ? "#ffffff" : "#000000",
+            "line-width": 3,
+            "line-opacity": 1,
+        },
+    });
+
     // Re-apply colors when vector tiles finish loading (they stream per viewport)
     let sourcedataTimer = null;
     map.on("sourcedata", (e) => {
@@ -885,6 +903,12 @@ function selectArea(code, dataKey) {
         selectedAreaCode = code;
     }
 
+    // When selecting a department from the dept-level list, also set selectedDeptCode
+    // so that switching to commune/postcode/section shows data for that dept
+    if (dataKey === "department") {
+        selectedDeptCode = selectedAreaCode;  // null if deselected, code if selected
+    }
+
     // Update list highlighting
     document.querySelectorAll(".area-item").forEach(el => {
         el.classList.toggle("selected", el.dataset.code === selectedAreaCode);
@@ -916,6 +940,7 @@ async function selectDeptForLevel(deptCode) {
     } else {
         selectedDeptCode = deptCode;
     }
+    searchedCommuneCode = null;  // Clear search highlight when manually selecting dept
 
     // Update list highlighting
     document.querySelectorAll(".area-item").forEach(el => {
@@ -964,13 +989,23 @@ function updateHighlight() {
         const hlId = `${LEVELS[i].dataKey}-highlight`;
         try { map.setLayoutProperty(hlId, "visibility", "none"); } catch (e) {}
     }
+    try { map.setLayoutProperty("dept-vector-highlight", "visibility", "none"); } catch (e) {}
 
-    // For dept-required levels: highlight the selected dept on the department highlight layer
+    // For dept-required levels: show dept boundary + optional searched commune
     if (LEVELS[activeLevel].needsDept && selectedDeptCode) {
+        // Use the detailed vector-tile dept boundary (not the simplified local GeoJSON)
         try {
-            map.setFilter("department-highlight", ["==", ["get", "code"], selectedDeptCode]);
-            map.setLayoutProperty("department-highlight", "visibility", "visible");
+            map.setFilter("dept-vector-highlight", ["==", ["get", "code"], selectedDeptCode]);
+            map.setLayoutProperty("dept-vector-highlight", "visibility", "visible");
         } catch (e) {}
+
+        // Also highlight the searched commune if at commune level
+        if (searchedCommuneCode && LEVELS[activeLevel].dataKey === "commune") {
+            try {
+                map.setFilter("commune-highlight", ["==", ["get", "code"], searchedCommuneCode]);
+                map.setLayoutProperty("commune-highlight", "visibility", "visible");
+            } catch (e) {}
+        }
         return;
     }
 
@@ -1096,19 +1131,43 @@ function closeSearchDropdown() {
     searchDropdownIdx = -1;
 }
 
-function onSelectPlace(commune) {
+async function onSelectPlace(commune) {
     closeSearchDropdown();
 
     const input = document.getElementById("place-search");
     input.value = commune.nom;
 
+    // Determine department code from the commune
+    const deptCode = commune.departement ? commune.departement.code : deptFromCode(commune.code);
+
+    // Switch to commune level and set the department
+    const communeLevelIdx = 3; // Commune is index 3 in LEVELS
+    activeLevel = communeLevelIdx;
+    selectedDeptCode = deptCode;
+    selectedAreaCode = null;
+    searchedCommuneCode = commune.code;
+
+    // Update radio button to reflect commune level
+    document.querySelectorAll('input[name="map-level"]')[activeLevel].checked = true;
+
+    // Load section data in the background (for potential drill-down)
+    loadSectionDataForDept(deptCode);
+
+    // Update all map visuals
+    updateLayerVisibility();
+    updateHighlight();
+    updatePriceRangeForLevel();
+    updateColors();
+    updateDynamicStat();
+    populateAreaList();
+
     // Fly to the commune
     if (commune.centre && commune.centre.coordinates) {
         const [lon, lat] = commune.centre.coordinates;
-        let zoom = 13;
+        let zoom = 12;
         if (commune.population > 500000) zoom = 11;
-        else if (commune.population > 100000) zoom = 12;
-        else if (commune.population > 20000) zoom = 12.5;
+        else if (commune.population > 100000) zoom = 11.5;
+        else if (commune.population > 20000) zoom = 12;
 
         map.flyTo({ center: [lon, lat], zoom, duration: 1500 });
     }
@@ -1178,6 +1237,12 @@ function initSearch() {
         if (query.length < 2) {
             closeSearchDropdown();
             lastResults = [];
+            // When input is cleared completely, reset search state
+            if (query.length === 0) {
+                searchedCommuneCode = null;
+                document.getElementById("place-details").classList.add("hidden");
+                updateHighlight();
+            }
             return;
         }
         debounceTimer = setTimeout(async () => {
@@ -1228,11 +1293,14 @@ function initSearch() {
 async function setLevel(levelIdx) {
     activeLevel = levelIdx;
     selectedAreaCode = null;  // Clear area selection
+    searchedCommuneCode = null;  // Clear search highlight
 
-    // If switching to a non-dept level, clear dept selection
-    if (!LEVELS[levelIdx].needsDept) {
-        selectedDeptCode = null;
-    }
+    // Note: selectedDeptCode is intentionally NOT cleared when switching levels.
+    // This preserves the department context so switching between levels
+    // (e.g. Commune → Dept → back to Commune) keeps the dept selection.
+
+    // Hide search details panel when switching levels
+    document.getElementById("place-details").classList.add("hidden");
 
     updateLayerVisibility();
     updateHighlight();
@@ -1317,6 +1385,7 @@ document.addEventListener("DOMContentLoaded", () => {
             try { map.setPaintProperty(`${level.dataKey}-highlight`, "line-color", hlColor); } catch (e) {}
             try { map.setPaintProperty(`${level.dataKey}-hover`, "line-color", hlColor); } catch (e) {}
         }
+        try { map.setPaintProperty("dept-vector-highlight", "line-color", hlColor); } catch (e) {}
     });
     // Restore saved theme (default: dark)
     const saved = localStorage.getItem("theme");