Upload depth_interactive_map_alpha.R

depth_interactive_map_alpha.R

@@ -204,15 +204,22 @@ FOSAS_COLS <- list(
   "cuerpos" = list(col = "cuerpos_total", breaks = c(1, 10, 30, 80, 200))
 )
 
-# CPV 2020 population sub-types -- per grid cell (codigo + nivel).
-# Breaks chosen on observed niv=4..8 ranges; map at any nivel is readable.
+# CPV 2020 population sub-types. The same column name (e.g. pobtot) lives in
+# both the per-cell grid join AND the muni-level aggregation, so step_expr can
+# read either; only the BREAKS differ between modes.
+#   breaks_grid -> niv 4..7 cells (chosen on observed CPV ranges per nivel)
+#   breaks_mun  -> per-municipality totals (CDMX/Monterrey/Guadalajara reach
+#                  the millions; smallest munis a few hundred)
 POP_COLS <- list(
   "POBTOT"     = list(col = "pobtot",     label = "Total population",
-                      breaks = c(100, 1000, 5000, 25000, 100000, 500000)),
+                      breaks_grid = c(100, 1000, 5000, 25000, 100000, 500000),
+                      breaks_mun  = c(5000, 25000, 100000, 250000, 500000, 1000000)),
   "PNACOP"     = list(col = "pnacop",     label = "Born abroad",
-                      breaks = c(5, 25, 100, 500, 2500, 10000)),
+                      breaks_grid = c(5, 25, 100, 500, 2500, 10000),
+                      breaks_mun  = c(25, 100, 500, 2500, 10000, 50000)),
   "PRESOP2015" = list(col = "presop2015", label = "Resident abroad (2015)",
-                      breaks = c(2, 10, 50, 250, 1000, 5000))
+                      breaks_grid = c(2, 10, 50, 250, 1000, 5000),
+                      breaks_mun  = c(10, 50, 250, 1000, 5000, 25000))
 )
 
 # PATP sub-types (kind controls paint family: "num" = step, "cat" = match).
@@ -374,6 +381,28 @@ WEB_SQL_TEMPLATES <- list(
                 FROM dim_pam_mun GROUP BY CVEGEO) p
        ON c.CVEGEO = p.CVEGEO",
 
+  # Municipal-level CPV 2020 totals. Aggregates at nivel 8 (cell ~1.3km cell
+  # length, ~224k cells) so even the smallest CDMX delegaciones are larger than
+  # any cell and don't get inflated by single-cell over-attribution.
+  src_pop_mun =
+    "WITH agg AS (
+       SELECT g.CVEGEO,
+              SUM(CASE WHEN cpv.pobtot     >= 0 THEN cpv.pobtot     ELSE 0 END) AS pobtot,
+              SUM(CASE WHEN cpv.pnacop     >= 0 THEN cpv.pnacop     ELSE 0 END) AS pnacop,
+              SUM(CASE WHEN cpv.presop2015 >= 0 THEN cpv.presop2015 ELSE 0 END) AS presop2015
+       FROM grid_nivel8 g
+       LEFT JOIN dim_pop_cpv_grid cpv
+         ON cpv.codigo = g.codigo AND cpv.nivel = 8
+       GROUP BY g.CVEGEO
+     )
+     SELECT c.municipio, c.entidad, c.CVEGEO,
+            COALESCE(a.pobtot,     0) AS pobtot,
+            COALESCE(a.pnacop,     0) AS pnacop,
+            COALESCE(a.presop2015, 0) AS presop2015,
+            c.wkt
+     FROM crime_mun c
+     LEFT JOIN agg a ON a.CVEGEO = c.CVEGEO",
+
   # patp: dim_patp_mun stores numeric values as VARCHAR; TRY_CAST converts
   # the 'n.d.' sentinels (and other non-numeric strings) to NULL.
   src_patp =
@@ -549,7 +578,10 @@ web_sql_qbreaks <- function(tbl, col, n_colors, filter_zero = TRUE) {
 # grid_crosswalk has codigo_n4..codigo_n8, so the equi-join always applies.
 # simplify_tol: degrees of simplification applied to g.wkt at query time
 # (0 = no simplification; default 0.005 deg ~= 500 m, suitable for N4-N6).
-web_sql_grid_unified <- function(niv, has_fosas, limit = NULL, simplify_tol = 0.005) {
+web_sql_grid_unified <- function(niv, has_fosas, limit = NULL, simplify_tol = 0.005,
+                                  has_cpv_grid = TRUE,
+                                  has_patp     = TRUE,
+                                  has_pobreza  = TRUE) {
   col_n        <- paste0("codigo_n", niv)
   limit_clause <- if (!is.null(limit)) paste0("\nLIMIT ", limit) else ""
   wkt_expr     <- if (simplify_tol > 0)
@@ -562,6 +594,40 @@ web_sql_grid_unified <- function(niv, has_fosas, limit = NULL, simplify_tol = 0.
   fosas_join <- if (has_fosas)
     "\n           LEFT JOIN dim_fosas_mun   f   ON g.CVEGEO = f.CVEGEO"
   else ""
+  # Each optional extension is rendered as (columns + join) pairs. If the
+  # backing table is missing on the deployed DB, we substitute NULLs for the
+  # columns and skip the join entirely -- the rest of the grid still loads.
+  cpv_cols <- if (has_cpv_grid)
+    "CASE WHEN cpv.pobtot     < 0 THEN NULL ELSE CAST(cpv.pobtot     AS DOUBLE) END AS pobtot,
+              CASE WHEN cpv.pnacop     < 0 THEN NULL ELSE CAST(cpv.pnacop     AS DOUBLE) END AS pnacop,
+              CASE WHEN cpv.presop2015 < 0 THEN NULL ELSE CAST(cpv.presop2015 AS DOUBLE) END AS presop2015"
+  else
+    "CAST(NULL AS DOUBLE) AS pobtot, CAST(NULL AS DOUBLE) AS pnacop, CAST(NULL AS DOUBLE) AS presop2015"
+  cpv_join <- if (has_cpv_grid)
+    paste0("\n       LEFT JOIN dim_pop_cpv_grid cpv ON cpv.codigo = g.codigo AND cpv.nivel = ", niv)
+  else ""
+  patp_cols <- if (has_patp)
+    "TRY_CAST(patp.t_part   AS DOUBLE) AS t_part,
+              TRY_CAST(patp.t_ocup   AS DOUBLE) AS t_ocup,
+              TRY_CAST(patp.pr       AS DOUBLE) AS pr,
+              CAST(patp.ind_conec    AS DOUBLE) AS ind_conec,
+              patp.pr_pred_loc_rur                          AS pr_pred_loc_rur,
+              patp.delitos_fc_b_j_a                         AS delitos_fc_b_j_a"
+  else
+    "CAST(NULL AS DOUBLE) AS t_part, CAST(NULL AS DOUBLE) AS t_ocup, CAST(NULL AS DOUBLE) AS pr,
+     CAST(NULL AS DOUBLE) AS ind_conec, CAST(NULL AS VARCHAR) AS pr_pred_loc_rur,
+     CAST(NULL AS VARCHAR) AS delitos_fc_b_j_a"
+  patp_join <- if (has_patp)
+    "\n       LEFT JOIN dim_patp_mun    patp ON patp.CVEGEO = g.CVEGEO"
+  else ""
+  pob_cols <- if (has_pobreza)
+    "CAST(pob.pobreza_pct     AS DOUBLE) AS pobreza_pct,
+              CAST(pob.pobreza_ext_pct AS DOUBLE) AS pobreza_ext_pct"
+  else
+    "CAST(NULL AS DOUBLE) AS pobreza_pct, CAST(NULL AS DOUBLE) AS pobreza_ext_pct"
+  pob_join <- if (has_pobreza)
+    "\n       LEFT JOIN dim_pobreza_mun pob  ON pob.CVEGEO = g.CVEGEO"
+  else ""
   # Subqueries ensure exactly one row per CVEGEO; direct joins on multi-row
   # tables (dim_desap_mun ~58k rows, dim_pam_mun ~7.5k rows) explode memory.
   desap_sub <- "(SELECT CVEGEO, AVG(final_desap_nl_per_100k) AS desap_rate FROM dim_desap_mun GROUP BY CVEGEO)"
@@ -608,26 +674,14 @@ web_sql_grid_unified <- function(niv, has_fosas, limit = NULL, simplify_tol = 0.
               CAST(c.secuestro       AS DOUBLE) AS secuestro,
               CAST(c.trafico_menores AS DOUBLE) AS trafico_menores,
               CAST(c.trata_personas  AS DOUBLE) AS trata_personas,
-              -- CPV 2020 per-cell population (joined by codigo + nivel); -6 is
-              -- INEGI's confidentiality suppression marker, normalised to NULL.
-              CASE WHEN cpv.pobtot     < 0 THEN NULL ELSE CAST(cpv.pobtot     AS DOUBLE) END AS pobtot,
-              CASE WHEN cpv.pnacop     < 0 THEN NULL ELSE CAST(cpv.pnacop     AS DOUBLE) END AS pnacop,
-              CASE WHEN cpv.presop2015 < 0 THEN NULL ELSE CAST(cpv.presop2015 AS DOUBLE) END AS presop2015,
+              {cpv_cols},
               COALESCE(CAST(d.desap_rate AS DOUBLE), 0) AS desap_per_100k{fosas_cols},
               COALESCE(r.giim_mun, 'No aplica')           AS giim_mun,
               COALESCE(CAST(p.pam_total AS INTEGER), 0)   AS pam_n,
               CAST(s.perception_insecurity_perc AS DOUBLE) AS insecurity_perc,
               CAST(s.dark_figures_perc          AS DOUBLE) AS dark_figures_perc,
-              -- patp_mun (numeric-as-VARCHAR -> TRY_CAST; categorical kept as string)
-              TRY_CAST(patp.t_part   AS DOUBLE) AS t_part,
-              TRY_CAST(patp.t_ocup   AS DOUBLE) AS t_ocup,
-              TRY_CAST(patp.pr       AS DOUBLE) AS pr,
-              CAST(patp.ind_conec    AS DOUBLE) AS ind_conec,
-              patp.pr_pred_loc_rur                          AS pr_pred_loc_rur,
-              patp.delitos_fc_b_j_a                         AS delitos_fc_b_j_a,
-              -- pobreza_mun (CONEVAL 2020)
-              CAST(pob.pobreza_pct     AS DOUBLE) AS pobreza_pct,
-              CAST(pob.pobreza_ext_pct AS DOUBLE) AS pobreza_ext_pct,
+              {patp_cols},
+              {pob_cols},
               COALESCE(pts.migrants_n, 0)                 AS migrants_n,
               COALESCE(pts.graves_n,   0)                 AS graves_n,
               COALESCE(pts.ocved_n,    0)                 AS ocved_n,
@@ -638,10 +692,7 @@ web_sql_grid_unified <- function(niv, has_fosas, limit = NULL, simplify_tol = 0.
        LEFT JOIN {desap_sub}     d    ON g.CVEGEO = d.CVEGEO{fosas_join}
        LEFT JOIN dim_remesas_mun r    ON g.CVEGEO = r.CVEGEO
        LEFT JOIN {pam_sub}       p    ON g.CVEGEO = p.CVEGEO
-       LEFT JOIN state_polygons  s    ON s.CVE_ENT = SUBSTR(g.CVEGEO, 1, 2)
-       LEFT JOIN dim_pop_cpv_grid cpv ON cpv.codigo = g.codigo AND cpv.nivel = {niv}
-       LEFT JOIN dim_patp_mun    patp ON patp.CVEGEO = g.CVEGEO
-       LEFT JOIN dim_pobreza_mun pob  ON pob.CVEGEO = g.CVEGEO
+       LEFT JOIN state_polygons  s    ON s.CVE_ENT = SUBSTR(g.CVEGEO, 1, 2){cpv_join}{patp_join}{pob_join}
        LEFT JOIN pts_src         pts  ON g.codigo = pts.codigo{limit_clause}"
   )
 }
@@ -859,11 +910,12 @@ WEB_SPEC <- list(
     list(id = "pam-choro",       type = "fill",   src = "src_pam",
          geom = "polygon", color = "pam_qstep",   value = "pam_n",
          opacity = 0.7,  visible = FALSE),
-    # pop-choro is GRID-ONLY: data lives in dim_pop_cpv_grid keyed by (codigo,
-    # nivel). The fill is painted directly on grid cells via add_typed_grid_layers,
-    # there is no mun-level fallback layer.
-    list(id = "pop-choro",       type = "fill",   src = NULL,
-         geom = "grid_only", color = "pop_qstep", value = "pobtot",
+    # pop-choro paints in BOTH modes:
+    #   non-grid: per-municipality aggregate of CPV cells (src_pop_mun)
+    #   grid:     per-cell value at the active nivel (dim_pop_cpv_grid, joined
+    #             into web_sql_grid_unified)
+    list(id = "pop-choro",       type = "fill",   src = "src_pop_mun",
+         geom = "polygon",   color = "pop_qstep", value = "pobtot",
          opacity = 0.7,  visible = FALSE),
     # patp-choro and pobreza-choro are also GRID-ONLY (joined into grid_unified)
     list(id = "patp-choro",      type = "fill",   src = NULL,
@@ -957,8 +1009,8 @@ WEB_SPEC <- list(
            tooltip = "Railway lines used by migrants, including freight routes known as 'La Bestia'.",
            legend = list(type = "line", color = "#9E2A3A", label = "Railway", dashed = TRUE)),
       list(kind = "layer", value = "pop-choro",
-           label = "Population per grid cell (CPV 2020)",
-           tooltip = "Census 2020 population counts joined to each rHEALPix grid cell at the active nivel. Enable the grid view to see this layer. Select sub-type: total population, born abroad, or resident abroad in 2015.",
+           label = "Population (CPV 2020)",
+           tooltip = "Census 2020 population counts. With the grid view off, each municipality shows the sum of its CPV cells (3 sub-types: total, born abroad, resident abroad in 2015). With the grid view on, each rHEALPix cell shows its own value at the active nivel (n4-n7).",
            legend = list(type = "ramp", pal = "pop", unit = " people")),
       list(kind = "radios", group = "pop")
     )),
@@ -2536,10 +2588,21 @@ server <- function(input, output, session) {
     if (!is.null(con)) DBI::dbDisconnect(con, shutdown = TRUE)
   })
 
-  # Check fosas table once at session start — gates all fosas SQL and JS calls
-  has_fosas_tbl <- if (!is.null(con)) {
-    tryCatch(DBI::dbExistsTable(con, "dim_fosas_mun"), error = function(e) FALSE)
-  } else FALSE
+  # Check optional tables once at session start. Each gates the SQL builders
+  # so a missing table does not cascade into the rest of the grid_unified
+  # query (which would otherwise blank all grid layers).
+  tbl_exists <- function(t) {
+    if (is.null(con)) return(FALSE)
+    tryCatch(DBI::dbExistsTable(con, t), error = function(e) FALSE)
+  }
+  has_fosas_tbl    <- tbl_exists("dim_fosas_mun")
+  has_cpv_grid_tbl <- tbl_exists("dim_pop_cpv_grid")
+  has_patp_tbl     <- tbl_exists("dim_patp_mun")
+  has_pobreza_tbl  <- tbl_exists("dim_pobreza_mun")
+  message("[deploy-check] dim_pop_cpv_grid=", has_cpv_grid_tbl,
+          "  dim_patp_mun=", has_patp_tbl,
+          "  dim_pobreza_mun=", has_pobreza_tbl,
+          "  dim_fosas_mun=", has_fosas_tbl)
 
   # Tracks whether grid N7 layers were successfully added to the map
   is_3d      <- reactiveVal(FALSE)
@@ -2664,12 +2727,13 @@ server <- function(input, output, session) {
     count_step_expr("pam_n", breaks, PAM_PAL)
   }
 
-  # CPV 2020 per-cell population. The sub-type (POBTOT / PNACOP / PRESOP2015)
-  # determines which column + break set drives the step paint. NULL cells
-  # (INEGI confidentiality suppression) fall to the lightest palette stop.
-  pop_grid_expr <- function(pop_type = "POBTOT") {
-    info <- POP_COLS[[pop_type]] %||% POP_COLS[["POBTOT"]]
-    step_expr(info$col, info$breaks, POP_PAL)
+  # CPV 2020 population paint. mode = "grid" reads per-cell values (joined into
+  # web_sql_grid_unified); mode = "mun" reads per-municipality aggregates
+  # (src_pop_mun). Same column name in both modes; only the breaks differ.
+  pop_grid_expr <- function(pop_type = "POBTOT", mode = "grid") {
+    info   <- POP_COLS[[pop_type]] %||% POP_COLS[["POBTOT"]]
+    breaks <- if (identical(mode, "mun")) info$breaks_mun else info$breaks_grid
+    step_expr(info$col, breaks, POP_PAL)
   }
 
   # Envipe on the GRID: each grid cell carries its containing state's
@@ -2737,8 +2801,8 @@ server <- function(input, output, session) {
       "desap-choro"    = isTRUE(lazy_added$desap),
       "remesas-choro"  = isTRUE(lazy_added$remesas),
       "pam-choro"      = isTRUE(lazy_added$pam),
-      # pop / patp / pobreza are grid-only -- no mun-level layer to gate.
-      "pop-choro"      = FALSE,
+      "pop-choro"      = isTRUE(lazy_added$pop),
+      # patp / pobreza remain grid-only -- no mun-level layer to gate.
       "patp-choro"     = FALSE,
       "pobreza-choro"  = FALSE,
       "fosas-choro"    = isTRUE(lazy_added$fosas),
@@ -2790,6 +2854,15 @@ server <- function(input, output, session) {
       )
     }
 
+    # Sync pop fill color to selected sub-type (POBTOT / PNACOP / PRESOP2015)
+    # when grid is OFF. Uses mun-mode breaks via pop_grid_expr(mode = "mun").
+    if (!grid_on && "pop-choro" %in% active && layer_ready("pop-choro")) {
+      prx <- prx |> set_paint_property(
+        "pop-choro", "fill-color",
+        pop_grid_expr(pop_type, "mun")
+      )
+    }
+
     for (lid in NON_GRID_LAYERS) {
       if (!layer_ready(lid)) next
       prx <- prx |> set_layout_property(
@@ -3114,7 +3187,10 @@ server <- function(input, output, session) {
       nv_tol <- if (niv >= 7L) 0.002 else 0.005
       tryCatch({
         DBI::dbGetQuery(con,
-          web_sql_grid_unified(niv, has_fosas_tbl, limit, simplify_tol = nv_tol)) |>
+          web_sql_grid_unified(niv, has_fosas_tbl, limit, simplify_tol = nv_tol,
+                               has_cpv_grid = has_cpv_grid_tbl,
+                               has_patp     = has_patp_tbl,
+                               has_pobreza  = has_pobreza_tbl)) |>
           sf::st_as_sf(wkt = "wkt", crs = 4326) |>
           dplyr::select(-wkt)
       }, error = function(e) { message("grid_nivel", niv, ": ", e$message); NULL })
@@ -3318,7 +3394,11 @@ server <- function(input, output, session) {
   observeEvent(input$grid_nivel, {
     req(input$grid_nivel == "7", !grid7_loaded(), !is.null(con))
     grid7_sf <<- tryCatch({
-      DBI::dbGetQuery(con, web_sql_grid_unified(7, has_fosas_tbl, simplify_tol = 0.002)) |>
+      DBI::dbGetQuery(con,
+        web_sql_grid_unified(7, has_fosas_tbl, simplify_tol = 0.002,
+                             has_cpv_grid = has_cpv_grid_tbl,
+                             has_patp     = has_patp_tbl,
+                             has_pobreza  = has_pobreza_tbl)) |>
         sf::st_as_sf(wkt = "wkt", crs = 4326) |>
         dplyr::select(-wkt)
     }, error = function(e) { message("N7 lazy load: ", e$message); NULL })
@@ -3387,12 +3467,21 @@ server <- function(input, output, session) {
     nv_tol   <- if (nv_chr == "7") 0.002 else 0.005
     sql      <- if (has_pre) paste("SELECT * FROM", pre_tbl)
                 else web_sql_grid_unified(as.integer(nv_chr), has_fosas_tbl,
-                                          simplify_tol = nv_tol)
+                                          simplify_tol = nv_tol,
+                                          has_cpv_grid = has_cpv_grid_tbl,
+                                          has_patp     = has_patp_tbl,
+                                          has_pobreza  = has_pobreza_tbl)
     sf_data <- tryCatch({
       DBI::dbGetQuery(con, sql) |>
         sf::st_as_sf(wkt = "wkt", crs = 4326) |>
         dplyr::select(-wkt)
-    }, error = function(e) { message("lazy grid n", nv_chr, ": ", e$message); NULL })
+    }, error = function(e) {
+      message("lazy grid n", nv_chr, ": ", e$message)
+      try(showNotification(paste0("Grid n", nv_chr, " load failed: ",
+                                  conditionMessage(e)),
+                            type = "error", duration = 12), silent = TRUE)
+      NULL
+    })
     if (!is.null(sf_data)) {
       maplibre_proxy("map") |> add_typed_grid_layers(sf_data, nv_chr)
       lazy_added[[key]] <<- TRUE
@@ -3427,7 +3516,14 @@ server <- function(input, output, session) {
             fill_color = pam_qstep_expr(), fill_opacity = 0.7,
             tooltip = list("concat", "<b>", list("get","municipio"), "</b>",
                            "<br>Irregular migration events: ", list("to-string", list("get","pam_n")))))
-      # pop-choro is GRID-ONLY (CPV 2020 per cell). No mun-level fallback.
+      if (!grid_on && "pop-choro"     %in% active && has_cpv_grid_tbl)
+        lazy_add_fill("pop",     "src_pop_mun", "pop-choro",     function(prx, d)
+          prx |> add_fill_layer("pop-choro",     source = d,
+            fill_color = pop_grid_expr("POBTOT", "mun"), fill_opacity = 0.7,
+            tooltip = list("concat", "<b>", list("get","municipio"), "</b>",
+                           "<br>Total pop (CPV): ", list("to-string", list("get","pobtot")),
+                           "<br>Born abroad: ",     list("to-string", list("get","pnacop")),
+                           "<br>Resident abroad 2015: ", list("to-string", list("get","presop2015")))))
       if (!grid_on && "fosas-choro"   %in% active && has_fosas_tbl)
         lazy_add_fill("fosas",   "src_fosas",   "fosas-choro",   function(prx, d)
           prx |> add_fill_layer("fosas-choro",   source = d,
@@ -3495,19 +3591,12 @@ server <- function(input, output, session) {
     )
     updateSelectInput(session, "state_select",
                       choices = c("Choose a state..." = "", state_ch))
-    # Pre-materialise grid joins for N4/N5/N6 into session TEMP TABLEs.
-    # Runs after national totals so the sidebar populates first.
-    # lazy_add_grid() will use these tables instead of re-running the full CTE.
-    for (.nv in c("4", "5", "6")) {
-      nv_tol <- 0.005
-      tryCatch(
-        DBI::dbExecute(con, paste0(
-          "CREATE TEMP TABLE IF NOT EXISTS grid_pre_n", .nv, " AS ",
-          web_sql_grid_unified(as.integer(.nv), has_fosas_tbl, simplify_tol = nv_tol)
-        )),
-        error = function(e) message("grid_pre_n", .nv, ": ", e$message)
-      )
-    }
+    # Pre-materialise grid joins for N4/N5/N6 -- TEMPORARILY DISABLED.
+    # The extended grid_unified CTE was blocking session startup on HF (3
+    # synchronous queries against a query with 9 LEFT JOINs). lazy_add_grid()
+    # now falls back to the direct CTE on first user click, which finishes
+    # within a few seconds and lets the user keep using the rest of the app.
+    # If this proves OK on HF, re-enable with a background/async strategy.
   }, once = TRUE)
 
   # Grid overlay on/off is driven explicitly by resolution clicks in the menu