Commit Β·
3dbff85
0
Parent(s):
deploy(l4): self-contained Riprap mirror
Browse filesThis view is limited to 50 files because it contains too many changes. Β See raw diff
- .env.example +34 -0
- .gitattributes +54 -0
- .github/ISSUE_TEMPLATE/bug_report.yml +65 -0
- .github/ISSUE_TEMPLATE/config.yml +8 -0
- .github/ISSUE_TEMPLATE/feature_request.yml +54 -0
- .github/ISSUE_TEMPLATE/port_to_new_city.yml +70 -0
- .github/PULL_REQUEST_TEMPLATE.md +38 -0
- .github/workflows/check.yml +43 -0
- .gitignore +89 -0
- CHANGELOG.md +96 -0
- CODE_OF_CONDUCT.md +85 -0
- CONTRIBUTING.md +127 -0
- Dockerfile +104 -0
- README.md +26 -0
- SECURITY.md +54 -0
- agent.py +52 -0
- app/__init__.py +0 -0
- app/areas/__init__.py +0 -0
- app/areas/nta.py +224 -0
- app/assets/__init__.py +0 -0
- app/assets/mta_entrances.py +73 -0
- app/assets/nycha.py +28 -0
- app/assets/schools.py +27 -0
- app/context/__init__.py +0 -0
- app/context/_polygonize.py +165 -0
- app/context/dob_permits.py +258 -0
- app/context/eo_chip_cache.py +345 -0
- app/context/floodnet.py +148 -0
- app/context/gliner_extract.py +147 -0
- app/context/microtopo.py +274 -0
- app/context/noaa_tides.py +110 -0
- app/context/npcc4_slr.py +42 -0
- app/context/nws_alerts.py +71 -0
- app/context/nws_obs.py +108 -0
- app/context/nyc311.py +161 -0
- app/context/terramind_nyc.py +485 -0
- app/context/terramind_synthesis.py +468 -0
- app/emissions.py +269 -0
- app/energy.py +56 -0
- app/flood_layers/__init__.py +0 -0
- app/flood_layers/dep_stormwater.py +168 -0
- app/flood_layers/ida_hwm.py +96 -0
- app/flood_layers/prithvi_live.py +563 -0
- app/flood_layers/prithvi_water.py +120 -0
- app/flood_layers/sandy_inundation.py +110 -0
- app/framing.py +249 -0
- app/fsm.py +1394 -0
- app/geocode.py +138 -0
- app/inference.py +268 -0
- app/intents/__init__.py +3 -0
.env.example
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Riprap environment configuration.
|
| 2 |
+
#
|
| 3 |
+
# Copy this file to `.env` and fill in the values that match the
|
| 4 |
+
# inference backend you want to talk to. The default profile runs
|
| 5 |
+
# only the app container, so both the LLM (vLLM serving Granite 4.1)
|
| 6 |
+
# and the ML specialist service must be reachable at HTTP endpoints.
|
| 7 |
+
#
|
| 8 |
+
# Three common configurations:
|
| 9 |
+
#
|
| 10 |
+
# 1. Easiest β talk to the live demo's backends. Adam runs a public
|
| 11 |
+
# MI300X droplet for the hackathon; if it's still up at demo time,
|
| 12 |
+
# both endpoints are reachable from anywhere.
|
| 13 |
+
#
|
| 14 |
+
# 2. Self-hosted β bring up your own MI300X droplet via
|
| 15 |
+
# docs/DROPLET-RUNBOOK.md, then point both URLs at it.
|
| 16 |
+
#
|
| 17 |
+
# 3. Full local β use `docker compose --profile with-models up` to
|
| 18 |
+
# run the riprap-models service yourself (requires a GPU on your
|
| 19 |
+
# box) and point a separate vLLM container at Granite 4.1.
|
| 20 |
+
|
| 21 |
+
# ---- Granite 4.1 reconciler (vLLM, OpenAI-compatible) -----------------
|
| 22 |
+
# Set to "ollama" instead of "vllm" if you have a local Ollama with
|
| 23 |
+
# granite4.1:8b pulled and want to use that.
|
| 24 |
+
RIPRAP_LLM_PRIMARY=vllm
|
| 25 |
+
RIPRAP_LLM_BASE_URL=http://your-vllm-host:8000/v1
|
| 26 |
+
RIPRAP_LLM_API_KEY=your-token-here
|
| 27 |
+
|
| 28 |
+
# ---- ML specialist service (Prithvi, TerraMind, GLiNER, etc.) ---------
|
| 29 |
+
RIPRAP_ML_BASE_URL=http://your-ml-host:7860
|
| 30 |
+
RIPRAP_ML_API_KEY=your-token-here
|
| 31 |
+
|
| 32 |
+
# ---- Backend pill labels (cosmetic, shown top-right of the UI) --------
|
| 33 |
+
RIPRAP_HARDWARE_LABEL=AMD MI300X
|
| 34 |
+
RIPRAP_ENGINE_LABEL=Granite 4.1 / vLLM
|
.gitattributes
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Riprap-specific LFS tracking
|
| 2 |
+
*.geojson filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.tif filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
*.pdf filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
# Pre-computed register paragraphs
|
| 6 |
+
data/registers/*.json filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
# Esri FileGDB internal binary files (DEP Stormwater scenario data)
|
| 8 |
+
*.gdbtable filter=lfs diff=lfs merge=lfs -text
|
| 9 |
+
*.gdbtablx filter=lfs diff=lfs merge=lfs -text
|
| 10 |
+
*.gdbindexes filter=lfs diff=lfs merge=lfs -text
|
| 11 |
+
*.atx filter=lfs diff=lfs merge=lfs -text
|
| 12 |
+
*.spx filter=lfs diff=lfs merge=lfs -text
|
| 13 |
+
*.freelist filter=lfs diff=lfs merge=lfs -text
|
| 14 |
+
*.horizon filter=lfs diff=lfs merge=lfs -text
|
| 15 |
+
*.FDO_UUID filter=lfs diff=lfs merge=lfs -text
|
| 16 |
+
# Hugging Face's standard LFS rules (kept for forward-compat with model assets)
|
| 17 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 18 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 19 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 20 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 21 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 22 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 23 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 24 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 25 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 26 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 28 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
| 29 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 30 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 31 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 32 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 33 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 34 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 35 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 41 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 42 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 43 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 44 |
+
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 45 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 46 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 47 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 48 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 49 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 50 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 51 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 52 |
+
*.pptx filter=lfs diff=lfs merge=lfs -text
|
| 53 |
+
assets/screenshots/** filter=lfs diff=lfs merge=lfs -text
|
| 54 |
+
slides/*.png filter=lfs diff=lfs merge=lfs -text
|
.github/ISSUE_TEMPLATE/bug_report.yml
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Bug report
|
| 2 |
+
description: A briefing came back wrong, a Stone failed to fire, or the UI broke.
|
| 3 |
+
title: "[bug] "
|
| 4 |
+
labels: ["bug"]
|
| 5 |
+
body:
|
| 6 |
+
- type: markdown
|
| 7 |
+
attributes:
|
| 8 |
+
value: |
|
| 9 |
+
Thanks for filing! Riprap is a hackathon-period demo; the more
|
| 10 |
+
reproducible the report, the faster it gets fixed.
|
| 11 |
+
- type: input
|
| 12 |
+
id: address
|
| 13 |
+
attributes:
|
| 14 |
+
label: NYC address tested
|
| 15 |
+
description: The exact string you typed (or "n/a" if the bug is UI-only).
|
| 16 |
+
placeholder: 80 Pioneer Street, Brooklyn
|
| 17 |
+
validations:
|
| 18 |
+
required: true
|
| 19 |
+
- type: textarea
|
| 20 |
+
id: expected
|
| 21 |
+
attributes:
|
| 22 |
+
label: Expected behavior
|
| 23 |
+
validations:
|
| 24 |
+
required: true
|
| 25 |
+
- type: textarea
|
| 26 |
+
id: actual
|
| 27 |
+
attributes:
|
| 28 |
+
label: Actual behavior
|
| 29 |
+
description: Paste the briefing text or describe the failure.
|
| 30 |
+
validations:
|
| 31 |
+
required: true
|
| 32 |
+
- type: dropdown
|
| 33 |
+
id: surface
|
| 34 |
+
attributes:
|
| 35 |
+
label: Where did you reproduce this?
|
| 36 |
+
options:
|
| 37 |
+
- Hosted demo (lablab Space)
|
| 38 |
+
- Local Docker (`docker compose up`)
|
| 39 |
+
- Local dev server (`uvicorn web.main:app`)
|
| 40 |
+
- Self-hosted GPU inference
|
| 41 |
+
validations:
|
| 42 |
+
required: true
|
| 43 |
+
- type: input
|
| 44 |
+
id: browser
|
| 45 |
+
attributes:
|
| 46 |
+
label: Browser / OS
|
| 47 |
+
placeholder: Chrome 142 on macOS 14
|
| 48 |
+
- type: textarea
|
| 49 |
+
id: console
|
| 50 |
+
attributes:
|
| 51 |
+
label: Browser console errors
|
| 52 |
+
description: DevTools β Console. Paste anything red.
|
| 53 |
+
render: text
|
| 54 |
+
- type: textarea
|
| 55 |
+
id: stream
|
| 56 |
+
attributes:
|
| 57 |
+
label: /api/agent/stream output (optional)
|
| 58 |
+
description: |
|
| 59 |
+
If the bug is a Stone failure, paste the relevant lines from the
|
| 60 |
+
SSE trace pane (or curl `/api/agent/stream?q=<address>` directly).
|
| 61 |
+
render: text
|
| 62 |
+
- type: textarea
|
| 63 |
+
id: notes
|
| 64 |
+
attributes:
|
| 65 |
+
label: Anything else
|
.github/ISSUE_TEMPLATE/config.yml
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
blank_issues_enabled: false
|
| 2 |
+
contact_links:
|
| 3 |
+
- name: Try the live demo
|
| 4 |
+
url: https://lablab-ai-amd-developer-hackathon-riprap-nyc.hf.space
|
| 5 |
+
about: Reproduce the issue against the hosted Space before filing.
|
| 6 |
+
- name: Read the architecture docs
|
| 7 |
+
url: https://github.com/msradam/riprap-nyc/tree/main/docs
|
| 8 |
+
about: ARCHITECTURE, METHODOLOGY, EMISSIONS, DEPLOY, BENCHMARKS, RESEARCH.
|
.github/ISSUE_TEMPLATE/feature_request.yml
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Feature request
|
| 2 |
+
description: Propose a new probe, a new Stone, or a new civic-tech use case.
|
| 3 |
+
title: "[feat] "
|
| 4 |
+
labels: ["enhancement"]
|
| 5 |
+
body:
|
| 6 |
+
- type: textarea
|
| 7 |
+
id: usecase
|
| 8 |
+
attributes:
|
| 9 |
+
label: Civic-tech use case
|
| 10 |
+
description: |
|
| 11 |
+
Who is the user, what decision are they making, and what
|
| 12 |
+
evidence would Riprap need to surface to support it?
|
| 13 |
+
placeholder: |
|
| 14 |
+
e.g. "A resilience office siting a capital project needs the
|
| 15 |
+
joint exposure of NYCHA + schools within 200m of a Sandy
|
| 16 |
+
100-year inundation polygon."
|
| 17 |
+
validations:
|
| 18 |
+
required: true
|
| 19 |
+
- type: textarea
|
| 20 |
+
id: data
|
| 21 |
+
attributes:
|
| 22 |
+
label: Data source(s)
|
| 23 |
+
description: |
|
| 24 |
+
Which public-record datasets should Riprap pull from? Include
|
| 25 |
+
URLs, agency owner, refresh cadence, and licence if known.
|
| 26 |
+
validations:
|
| 27 |
+
required: true
|
| 28 |
+
- type: dropdown
|
| 29 |
+
id: stone
|
| 30 |
+
attributes:
|
| 31 |
+
label: Which Stone does this belong in?
|
| 32 |
+
options:
|
| 33 |
+
- Cornerstone (hazard memory)
|
| 34 |
+
- Keystone (asset registers)
|
| 35 |
+
- Touchstone (live observation)
|
| 36 |
+
- Lodestone (forecast)
|
| 37 |
+
- Capstone (synthesis)
|
| 38 |
+
- Not sure / cross-cutting
|
| 39 |
+
validations:
|
| 40 |
+
required: true
|
| 41 |
+
- type: dropdown
|
| 42 |
+
id: contribute
|
| 43 |
+
attributes:
|
| 44 |
+
label: Willing to contribute the implementation?
|
| 45 |
+
options:
|
| 46 |
+
- "Yes β I can open the PR"
|
| 47 |
+
- "Maybe β with mentorship"
|
| 48 |
+
- "No β flagging the gap"
|
| 49 |
+
validations:
|
| 50 |
+
required: true
|
| 51 |
+
- type: textarea
|
| 52 |
+
id: notes
|
| 53 |
+
attributes:
|
| 54 |
+
label: Anything else
|
.github/ISSUE_TEMPLATE/port_to_new_city.yml
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Port to a new city
|
| 2 |
+
description: Plan a Riprap deployment for a city other than NYC.
|
| 3 |
+
title: "[port] "
|
| 4 |
+
labels: ["port", "enhancement"]
|
| 5 |
+
body:
|
| 6 |
+
- type: markdown
|
| 7 |
+
attributes:
|
| 8 |
+
value: |
|
| 9 |
+
Riprap's Five Stones taxonomy is city-agnostic; only the probes
|
| 10 |
+
plugged into each Stone change. See the "Five Stones beyond NYC"
|
| 11 |
+
section in the README. This template helps scope a port.
|
| 12 |
+
- type: input
|
| 13 |
+
id: city
|
| 14 |
+
attributes:
|
| 15 |
+
label: Target city / region
|
| 16 |
+
placeholder: e.g. Houston, TX
|
| 17 |
+
validations:
|
| 18 |
+
required: true
|
| 19 |
+
- type: textarea
|
| 20 |
+
id: cornerstone
|
| 21 |
+
attributes:
|
| 22 |
+
label: Cornerstone β hazard memory
|
| 23 |
+
description: |
|
| 24 |
+
Local historical inundation extents, regional DEM, regulatory
|
| 25 |
+
floodplain maps. Include dataset URLs and licences.
|
| 26 |
+
validations:
|
| 27 |
+
required: true
|
| 28 |
+
- type: textarea
|
| 29 |
+
id: keystone
|
| 30 |
+
attributes:
|
| 31 |
+
label: Keystone β asset registers
|
| 32 |
+
description: |
|
| 33 |
+
Transit, housing, education, healthcare polygons your jurisdiction
|
| 34 |
+
publishes.
|
| 35 |
+
validations:
|
| 36 |
+
required: true
|
| 37 |
+
- type: textarea
|
| 38 |
+
id: touchstone
|
| 39 |
+
attributes:
|
| 40 |
+
label: Touchstone β live observation
|
| 41 |
+
description: |
|
| 42 |
+
Live sensors, complaint streams (e.g. Houston has FloodNet
|
| 43 |
+
analogues; many cities expose 311 or equivalent).
|
| 44 |
+
validations:
|
| 45 |
+
required: true
|
| 46 |
+
- type: textarea
|
| 47 |
+
id: lodestone
|
| 48 |
+
attributes:
|
| 49 |
+
label: Lodestone β forecast
|
| 50 |
+
description: |
|
| 51 |
+
Local NWS / hydrologic / surge models, tide gauges, time-series
|
| 52 |
+
fine-tunes you'd retrain.
|
| 53 |
+
validations:
|
| 54 |
+
required: true
|
| 55 |
+
- type: dropdown
|
| 56 |
+
id: hardware
|
| 57 |
+
attributes:
|
| 58 |
+
label: Target inference hardware
|
| 59 |
+
options:
|
| 60 |
+
- AMD MI300X (or other ROCm)
|
| 61 |
+
- NVIDIA L4 / A10
|
| 62 |
+
- NVIDIA H100 / A100
|
| 63 |
+
- CPU-only (Ollama)
|
| 64 |
+
- Not decided
|
| 65 |
+
validations:
|
| 66 |
+
required: true
|
| 67 |
+
- type: textarea
|
| 68 |
+
id: notes
|
| 69 |
+
attributes:
|
| 70 |
+
label: Anything else
|
.github/PULL_REQUEST_TEMPLATE.md
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<!-- Thanks for opening a PR. The checklist below mirrors how Riprap
|
| 2 |
+
was kept stable through hackathon week. -->
|
| 3 |
+
|
| 4 |
+
## Summary
|
| 5 |
+
|
| 6 |
+
<!-- One paragraph: what changed, and why. Reference issues with #N. -->
|
| 7 |
+
|
| 8 |
+
## Tested against
|
| 9 |
+
|
| 10 |
+
- [ ] Local dev server (`uvicorn web.main:app`)
|
| 11 |
+
- [ ] Local Docker (`docker compose up`)
|
| 12 |
+
- [ ] Hosted lablab Space
|
| 13 |
+
- [ ] Self-hosted GPU inference
|
| 14 |
+
|
| 15 |
+
## Stones-fire probe
|
| 16 |
+
|
| 17 |
+
<!-- Paste the tail of `scripts/probe_stones_fire.py` output. The PR
|
| 18 |
+
should not be merged unless all five Stones fire. -->
|
| 19 |
+
|
| 20 |
+
```
|
| 21 |
+
PYTHONPATH=. uv run python scripts/probe_stones_fire.py --timeout 600
|
| 22 |
+
```
|
| 23 |
+
|
| 24 |
+
## Energy-ledger sanity check
|
| 25 |
+
|
| 26 |
+
<!-- If this PR touches inference, app/emissions.py, or app/llm.py:
|
| 27 |
+
paste the n_measured / n_calls ratio and confirm hardware label. -->
|
| 28 |
+
|
| 29 |
+
## Checklist
|
| 30 |
+
|
| 31 |
+
- [ ] No regression in `app/`, `web/`, `services/`, or
|
| 32 |
+
`inference-vllm/proxy.py` logic (typo-only edits OK).
|
| 33 |
+
- [ ] Docs updated (`README.md`, relevant `docs/*.md`) if public
|
| 34 |
+
surface changed.
|
| 35 |
+
- [ ] `CHANGELOG.md` entry under `[Unreleased]` with the right
|
| 36 |
+
`Added` / `Changed` / `Fixed` bucket.
|
| 37 |
+
- [ ] Conventional-commit prefix on the squash title
|
| 38 |
+
(`feat:` / `fix:` / `docs:` / `chore:` / `build:`).
|
.github/workflows/check.yml
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: check
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
push:
|
| 5 |
+
branches: [main]
|
| 6 |
+
pull_request:
|
| 7 |
+
branches: [main]
|
| 8 |
+
|
| 9 |
+
permissions:
|
| 10 |
+
contents: read
|
| 11 |
+
|
| 12 |
+
jobs:
|
| 13 |
+
check:
|
| 14 |
+
name: import + lightweight tests
|
| 15 |
+
runs-on: ubuntu-latest
|
| 16 |
+
timeout-minutes: 15
|
| 17 |
+
steps:
|
| 18 |
+
- uses: actions/checkout@v4
|
| 19 |
+
|
| 20 |
+
- name: Install uv
|
| 21 |
+
uses: astral-sh/setup-uv@v3
|
| 22 |
+
with:
|
| 23 |
+
enable-cache: true
|
| 24 |
+
|
| 25 |
+
- name: Set up Python 3.12
|
| 26 |
+
run: uv python install 3.12
|
| 27 |
+
|
| 28 |
+
- name: Create venv and install deps
|
| 29 |
+
run: |
|
| 30 |
+
uv venv --python 3.12
|
| 31 |
+
uv pip install -r requirements.txt
|
| 32 |
+
|
| 33 |
+
- name: Import smoke test
|
| 34 |
+
env:
|
| 35 |
+
PYTHONPATH: .
|
| 36 |
+
run: |
|
| 37 |
+
uv run python -c "from app import fsm, llm, inference, emissions; from web import main"
|
| 38 |
+
|
| 39 |
+
- name: Lightweight pytest subset
|
| 40 |
+
env:
|
| 41 |
+
PYTHONPATH: .
|
| 42 |
+
run: |
|
| 43 |
+
uv run pytest -q tests/test_stones.py tests/test_compare_shape.py tests/test_stone_envelope.py
|
.gitignore
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Session artifacts (legacy agent reports β not for the public repo)
|
| 2 |
+
*MORNING-BRIEF*.md
|
| 3 |
+
*OVERNIGHT*.md
|
| 4 |
+
*COMMS-OVERNIGHT*.md
|
| 5 |
+
CODE-MORNING-BRIEF*.md
|
| 6 |
+
MONDAY.md
|
| 7 |
+
FRIDAY*.md
|
| 8 |
+
*-REPORT.md
|
| 9 |
+
docs/sessions/
|
| 10 |
+
docs/design_handoff/
|
| 11 |
+
|
| 12 |
+
# Local-only secrets / credentials
|
| 13 |
+
AMD_TOKEN
|
| 14 |
+
|
| 15 |
+
# Probe / batch / diagnostic output (regenerable; not for the repo)
|
| 16 |
+
tests/batch_results.json
|
| 17 |
+
tests/overnight_audit.json
|
| 18 |
+
scripts/diagnostic_*.py
|
| 19 |
+
scripts/find_top_locations.py
|
| 20 |
+
scripts/verify_locations.py
|
| 21 |
+
|
| 22 |
+
__pycache__/
|
| 23 |
+
*.py[cod]
|
| 24 |
+
*.egg-info/
|
| 25 |
+
dist/
|
| 26 |
+
build/
|
| 27 |
+
.venv/
|
| 28 |
+
.env
|
| 29 |
+
.DS_Store
|
| 30 |
+
outputs/
|
| 31 |
+
node_modules/
|
| 32 |
+
*.tmp
|
| 33 |
+
*.log
|
| 34 |
+
.ruff_cache/
|
| 35 |
+
.pytest_cache/
|
| 36 |
+
.ipynb_checkpoints/
|
| 37 |
+
|
| 38 |
+
# Claude Code context (per-machine, not for the public repo)
|
| 39 |
+
CLAUDE.md
|
| 40 |
+
CLAUDE.local.md
|
| 41 |
+
.claude/
|
| 42 |
+
|
| 43 |
+
# legacy / intermediate Prithvi artifacts (not shipped)
|
| 44 |
+
data/hls_stack_*.tif
|
| 45 |
+
data/prithvi_runs/
|
| 46 |
+
data/*.legacy_*
|
| 47 |
+
web/svelte/node_modules/
|
| 48 |
+
web/sveltekit/node_modules/
|
| 49 |
+
web/sveltekit/.svelte-kit/
|
| 50 |
+
# web/sveltekit/build/ (uncommented to allow deployment to HF Space)
|
| 51 |
+
# web/sveltekit/build/
|
| 52 |
+
|
| 53 |
+
# Experiments β cached HF model downloads, training artifacts, intermediate
|
| 54 |
+
# fixtures. RESULTS.md, NOTES.md, and source code stay tracked.
|
| 55 |
+
experiments/**/.cache/
|
| 56 |
+
experiments/**/restore/
|
| 57 |
+
experiments/**/publish/
|
| 58 |
+
experiments/**/*.tif
|
| 59 |
+
experiments/**/*.png
|
| 60 |
+
experiments/**/*.jpg
|
| 61 |
+
experiments/**/*.parquet
|
| 62 |
+
experiments/**/*.npy
|
| 63 |
+
pitch/screenshots-*/
|
| 64 |
+
|
| 65 |
+
# Marp deck render artefacts (regenerable via `make` in slides/)
|
| 66 |
+
slides/deck.pdf
|
| 67 |
+
slides/deck.html
|
| 68 |
+
slides/deck.pptx
|
| 69 |
+
|
| 70 |
+
# Session artifacts
|
| 71 |
+
/tmp/riprap-*
|
| 72 |
+
.deploy-state
|
| 73 |
+
*.bak
|
| 74 |
+
*.swp
|
| 75 |
+
*.swo
|
| 76 |
+
.playwright-mcp/
|
| 77 |
+
|
| 78 |
+
# Demo recordings (large; not committed)
|
| 79 |
+
assets/video/
|
| 80 |
+
slides/*.mp4
|
| 81 |
+
slides/asce/speaker_notes.md
|
| 82 |
+
|
| 83 |
+
# Local env overlays
|
| 84 |
+
.env.local
|
| 85 |
+
*.local.env
|
| 86 |
+
|
| 87 |
+
# Sensitive
|
| 88 |
+
AMD_TOKEN
|
| 89 |
+
submission.md
|
CHANGELOG.md
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Changelog
|
| 2 |
+
|
| 3 |
+
All notable changes to Riprap. The hackathon submission tag is
|
| 4 |
+
`v0.5.0` (build 2026-05-07); subsequent dates record polish work
|
| 5 |
+
that landed on the hackathon-period production deploys.
|
| 6 |
+
|
| 7 |
+
## [Unreleased] β 2026-05-09 (Saturday)
|
| 8 |
+
|
| 9 |
+
### Added
|
| 10 |
+
- **Per-query inference energy ledger** with real NVML readings off
|
| 11 |
+
the L4 GPU. The status row on the Findings region now reports
|
| 12 |
+
total Wh + total tokens for every briefing, with a leading icon
|
| 13 |
+
(`β` / `β` / `~`) disclosing whether the number was measured or
|
| 14 |
+
estimated. Full breakdown documented in
|
| 15 |
+
[`docs/EMISSIONS.md`](docs/EMISSIONS.md).
|
| 16 |
+
- `inference-vllm/proxy.py`: 100 ms-cadence NVML sampler, response
|
| 17 |
+
headers `X-GPU-Power-W` / `X-GPU-Energy-J` on every forwarded
|
| 18 |
+
POST, and a `GET /v1/power` endpoint for bracket-sampling clients.
|
| 19 |
+
- `app/emissions.py` β new module with a thread-local `Tracker` that
|
| 20 |
+
records every LLM and ML inference call (model, hardware, tokens,
|
| 21 |
+
duration, joules) with a `measured: bool` flag per row.
|
| 22 |
+
- `scripts/probe_stones_fire.py` β programmatic CI that runs an
|
| 23 |
+
address query against the lablab UI and asserts all five Stones
|
| 24 |
+
fire, no `torchvision::nms` / `deps unavailable` dep regression,
|
| 25 |
+
and the `emissions` block carries `nvidia_l4` hardware.
|
| 26 |
+
- `scripts/probe_benchmarks.py` β collects the canonical
|
| 27 |
+
four-address verification set into `outputs/benchmarks.json`
|
| 28 |
+
for the `docs/BENCHMARKS.md` page.
|
| 29 |
+
- `docs/EMISSIONS.md`, `docs/DEPLOY.md`, `docs/BENCHMARKS.md`,
|
| 30 |
+
`CHANGELOG.md`, `CONTRIBUTING.md`.
|
| 31 |
+
|
| 32 |
+
### Changed
|
| 33 |
+
- The `RunHealthStrip` chip dropped the cloud-energy comparison
|
| 34 |
+
(the sign convention was misleading and the comparison is now
|
| 35 |
+
redundant given real measurements). New format:
|
| 36 |
+
`<icon> X.X Wh / Y.YK tok inference`.
|
| 37 |
+
- `app/llm.py:_default_hardware_label` defaults to `"NVIDIA L4"`
|
| 38 |
+
when remote vLLM is configured (was `"AMD MI300X"`, a stale
|
| 39 |
+
string from the droplet days).
|
| 40 |
+
- `app/llm.py:chat()` now brackets every completion with two GETs
|
| 41 |
+
to the inference Space's `/v1/power` endpoint; the average powers
|
| 42 |
+
the LLM-call energy reading instead of the data-sheet estimate.
|
| 43 |
+
- `app/inference.py:_post()` reads NVML headers off the proxy
|
| 44 |
+
response and forwards real joules into `emissions.record_ml`.
|
| 45 |
+
|
| 46 |
+
### Fixed
|
| 47 |
+
- `app/flood_layers/prithvi_live.py`: when the configured remote
|
| 48 |
+
inference call fails (`RemoteUnreachable`), the specialist no
|
| 49 |
+
longer falls through to the local terratorch path. The local
|
| 50 |
+
path crashes with `RuntimeError: operator torchvision::nms does
|
| 51 |
+
not exist` on the cpu-basic UI Space; surfacing a clean
|
| 52 |
+
`remote prithvi-pluvial unreachable` skip is correct.
|
| 53 |
+
- `app/context/terramind_nyc.py:_try_remote()`: returns a
|
| 54 |
+
`{"ok": False, "skipped": "remote terramind/<adapter>: ..."}`
|
| 55 |
+
sentinel on remote failure, instead of `None` which was
|
| 56 |
+
silently masked as `deps unavailable on this deployment`.
|
| 57 |
+
- `web/main.py`: explicit `/favicon.svg`, `/favicon.png`,
|
| 58 |
+
`/favicon.ico`, `/robots.txt` routes β they were 404-ing under
|
| 59 |
+
the SvelteKit SPA fallback because only `/_app` was mounted off
|
| 60 |
+
the build directory.
|
| 61 |
+
|
| 62 |
+
### Documentation
|
| 63 |
+
- Full README rewrite reflecting the post-droplet L4 topology, the
|
| 64 |
+
new emissions feature, and updated repo structure. Hackathon
|
| 65 |
+
framing preserved.
|
| 66 |
+
- New `docs/DEPLOY.md` with the production topology, env-var
|
| 67 |
+
reference, and per-Space deploy commands.
|
| 68 |
+
- New `docs/EMISSIONS.md` documenting what's measured vs. estimated,
|
| 69 |
+
the NVML pipeline, and how to verify.
|
| 70 |
+
|
| 71 |
+
### Infrastructure note
|
| 72 |
+
- The DigitalOcean MI300X droplet was decommissioned 2026-05-06.
|
| 73 |
+
All production inference now serves from `msradam/riprap-vllm`
|
| 74 |
+
(NVIDIA L4). The MI300X runbook is preserved in
|
| 75 |
+
[`docs/DROPLET-RUNBOOK.md`](docs/DROPLET-RUNBOOK.md) for anyone
|
| 76 |
+
reproducing the AMD-judging setup; setting
|
| 77 |
+
`RIPRAP_HARDWARE_LABEL=AMD MI300X` swaps the emissions profile
|
| 78 |
+
back when redeploying to that hardware.
|
| 79 |
+
|
| 80 |
+
---
|
| 81 |
+
|
| 82 |
+
## [v0.5.0] β 2026-05-07
|
| 83 |
+
|
| 84 |
+
Hackathon submission tag.
|
| 85 |
+
|
| 86 |
+
### Added
|
| 87 |
+
- Five-Stone Burr FSM with Granite-native document-role messages
|
| 88 |
+
- Mellea four-check rejection sampling for the Capstone
|
| 89 |
+
- SvelteKit UI with SSE streaming, briefing prose, evidence-card
|
| 90 |
+
grid, MapLibre overlay, citation drawer
|
| 91 |
+
- Three NYC-specialised foundation models published Apache-2.0:
|
| 92 |
+
`msradam/TerraMind-NYC-Adapters` (LULC + Buildings + TiM LoRAs),
|
| 93 |
+
`msradam/Prithvi-EO-2.0-NYC-Pluvial`,
|
| 94 |
+
`msradam/Granite-TTM-r2-Battery-Surge`
|
| 95 |
+
- 30+ FSM specialists across hazard memory, asset registers, live
|
| 96 |
+
observation, forecasting, and citation-grounded synthesis
|
CODE_OF_CONDUCT.md
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
# Contributor Covenant Code of Conduct
|
| 3 |
+
|
| 4 |
+
## Our Pledge
|
| 5 |
+
|
| 6 |
+
We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, caste, color, religion, or sexual identity and orientation.
|
| 7 |
+
|
| 8 |
+
We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community.
|
| 9 |
+
|
| 10 |
+
## Our Standards
|
| 11 |
+
|
| 12 |
+
Examples of behavior that contributes to a positive environment for our community include:
|
| 13 |
+
|
| 14 |
+
* Demonstrating empathy and kindness toward other people
|
| 15 |
+
* Being respectful of differing opinions, viewpoints, and experiences
|
| 16 |
+
* Giving and gracefully accepting constructive feedback
|
| 17 |
+
* Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience
|
| 18 |
+
* Focusing on what is best not just for us as individuals, but for the overall community
|
| 19 |
+
|
| 20 |
+
Examples of unacceptable behavior include:
|
| 21 |
+
|
| 22 |
+
* The use of sexualized language or imagery, and sexual attention or advances of any kind
|
| 23 |
+
* Trolling, insulting or derogatory comments, and personal or political attacks
|
| 24 |
+
* Public or private harassment
|
| 25 |
+
* Publishing others' private information, such as a physical or email address, without their explicit permission
|
| 26 |
+
* Other conduct which could reasonably be considered inappropriate in a professional setting
|
| 27 |
+
|
| 28 |
+
## Enforcement Responsibilities
|
| 29 |
+
|
| 30 |
+
Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful.
|
| 31 |
+
|
| 32 |
+
Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate.
|
| 33 |
+
|
| 34 |
+
## Scope
|
| 35 |
+
|
| 36 |
+
This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event.
|
| 37 |
+
|
| 38 |
+
## Enforcement
|
| 39 |
+
|
| 40 |
+
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at msrahmanadam@gmail.com. All complaints will be reviewed and investigated promptly and fairly.
|
| 41 |
+
|
| 42 |
+
All community leaders are obligated to respect the privacy and security of the reporter of any incident.
|
| 43 |
+
|
| 44 |
+
## Enforcement Guidelines
|
| 45 |
+
|
| 46 |
+
Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct:
|
| 47 |
+
|
| 48 |
+
### 1. Correction
|
| 49 |
+
|
| 50 |
+
**Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community.
|
| 51 |
+
|
| 52 |
+
**Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested.
|
| 53 |
+
|
| 54 |
+
### 2. Warning
|
| 55 |
+
|
| 56 |
+
**Community Impact**: A violation through a single incident or series of actions.
|
| 57 |
+
|
| 58 |
+
**Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban.
|
| 59 |
+
|
| 60 |
+
### 3. Temporary Ban
|
| 61 |
+
|
| 62 |
+
**Community Impact**: A serious violation of community standards, including sustained inappropriate behavior.
|
| 63 |
+
|
| 64 |
+
**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban.
|
| 65 |
+
|
| 66 |
+
### 4. Permanent Ban
|
| 67 |
+
|
| 68 |
+
**Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals.
|
| 69 |
+
|
| 70 |
+
**Consequence**: A permanent ban from any sort of public interaction within the community.
|
| 71 |
+
|
| 72 |
+
## Attribution
|
| 73 |
+
|
| 74 |
+
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.1, available at [https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
|
| 75 |
+
|
| 76 |
+
Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder][Mozilla CoC].
|
| 77 |
+
|
| 78 |
+
For answers to common questions about this code of conduct, see the FAQ at [https://www.contributor-covenant.org/faq][FAQ]. Translations are available at [https://www.contributor-covenant.org/translations][translations].
|
| 79 |
+
|
| 80 |
+
[homepage]: https://www.contributor-covenant.org
|
| 81 |
+
[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
|
| 82 |
+
[Mozilla CoC]: https://github.com/mozilla/diversity
|
| 83 |
+
[FAQ]: https://www.contributor-covenant.org/faq
|
| 84 |
+
[translations]: https://www.contributor-covenant.org/translations
|
| 85 |
+
|
CONTRIBUTING.md
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Contributing
|
| 2 |
+
|
| 3 |
+
Riprap is the hackathon submission for the AMD Γ lablab.ai
|
| 4 |
+
Developer Hackathon, but the source ships under Apache 2.0 and is
|
| 5 |
+
intended to be reusable as a template for citation-grounded civic
|
| 6 |
+
AI in any flood-vulnerable region. Pull requests welcome.
|
| 7 |
+
|
| 8 |
+
## Quickstart
|
| 9 |
+
|
| 10 |
+
Python 3.12 + `uv`:
|
| 11 |
+
|
| 12 |
+
```bash
|
| 13 |
+
git clone https://github.com/msradam/riprap-nyc
|
| 14 |
+
cd riprap-nyc
|
| 15 |
+
uv venv && uv pip install -r requirements.txt
|
| 16 |
+
```
|
| 17 |
+
|
| 18 |
+
SvelteKit (the build is committed; only rebuild when sources
|
| 19 |
+
change under `web/sveltekit/src`):
|
| 20 |
+
|
| 21 |
+
```bash
|
| 22 |
+
cd web/sveltekit && npm ci && npm run build && cd ../..
|
| 23 |
+
```
|
| 24 |
+
|
| 25 |
+
Run the dev server locally pointing at the production inference
|
| 26 |
+
Space (real Granite + EO models, real NVML energy readings):
|
| 27 |
+
|
| 28 |
+
```bash
|
| 29 |
+
RIPRAP_LLM_PRIMARY=vllm \
|
| 30 |
+
RIPRAP_LLM_BASE_URL=https://msradam-riprap-vllm.hf.space/v1 \
|
| 31 |
+
RIPRAP_LLM_API_KEY=<token> \
|
| 32 |
+
RIPRAP_ML_BACKEND=remote \
|
| 33 |
+
RIPRAP_ML_BASE_URL=https://msradam-riprap-vllm.hf.space \
|
| 34 |
+
RIPRAP_ML_API_KEY=<token> \
|
| 35 |
+
.venv/bin/uvicorn web.main:app --host 127.0.0.1 --port 7860
|
| 36 |
+
```
|
| 37 |
+
|
| 38 |
+
Or run pure-local with Ollama (no GPU readings; data-sheet estimate):
|
| 39 |
+
|
| 40 |
+
```bash
|
| 41 |
+
ollama pull granite4.1:3b granite4.1:8b
|
| 42 |
+
.venv/bin/uvicorn web.main:app --host 127.0.0.1 --port 7860
|
| 43 |
+
```
|
| 44 |
+
|
| 45 |
+
## Verifying changes
|
| 46 |
+
|
| 47 |
+
Two probe scripts exercise the live deployment end-to-end:
|
| 48 |
+
|
| 49 |
+
```bash
|
| 50 |
+
# All five Stones must fire on the canonical address; emissions
|
| 51 |
+
# block must carry nvidia_l4 hardware; no torchvision/terratorch
|
| 52 |
+
# dep regressions in the trace.
|
| 53 |
+
PYTHONPATH=. uv run python scripts/probe_stones_fire.py --timeout 600
|
| 54 |
+
|
| 55 |
+
# Full canonical suite β five NYC addresses, intent-aware checks,
|
| 56 |
+
# Mellea grounding budget, no specialist crashes.
|
| 57 |
+
.venv/bin/python scripts/probe_addresses.py \
|
| 58 |
+
--base https://lablab-ai-amd-developer-hackathon-riprap-nyc.hf.space
|
| 59 |
+
```
|
| 60 |
+
|
| 61 |
+
Both default to the lablab UI Space; pass `--base http://127.0.0.1:7860`
|
| 62 |
+
to hit a local server.
|
| 63 |
+
|
| 64 |
+
## Structure
|
| 65 |
+
|
| 66 |
+
```
|
| 67 |
+
app/ Python package β the FSM and its specialists
|
| 68 |
+
βββ fsm.py Burr FSM, one @action per probe
|
| 69 |
+
βββ llm.py LiteLLM Router shim (Ollama / vLLM)
|
| 70 |
+
βββ inference.py HTTP client for the riprap-models service
|
| 71 |
+
βββ emissions.py Per-query energy + token tracker
|
| 72 |
+
βββ stones/ Stone taxonomy (NAME / TAGLINE / collect())
|
| 73 |
+
βββ flood_layers/ Cornerstone probes (sandy, dep, microtopo, β¦)
|
| 74 |
+
βββ context/ Keystone + Touchstone register + EO probes
|
| 75 |
+
βββ live/ Lodestone forecast probes
|
| 76 |
+
βββ intents/ single_address / neighborhood / compare / live_now
|
| 77 |
+
βββ reconcile.py Capstone β Granite-native document reconcile
|
| 78 |
+
βββ mellea_validator.py Mellea four-check rejection sampling
|
| 79 |
+
|
| 80 |
+
web/ FastAPI + SvelteKit
|
| 81 |
+
βββ main.py FastAPI app, SSE streaming, layer endpoints
|
| 82 |
+
βββ sveltekit/ Primary UI (adapter-static; build committed)
|
| 83 |
+
βββ static/ Legacy custom-element pages (still mounted)
|
| 84 |
+
|
| 85 |
+
inference-vllm/ Inference Space source (vLLM + EO models + proxy)
|
| 86 |
+
βββ Dockerfile L4 image, bakes Granite 4.1 8B FP8 + EO deps
|
| 87 |
+
βββ entrypoint.sh Boots vllm, riprap-models, proxy as subprocesses
|
| 88 |
+
βββ proxy.py Bearer-auth + NVML power sampler + SSE pass-through
|
| 89 |
+
|
| 90 |
+
inference/ Ollama-backed inference Space (fallback variant)
|
| 91 |
+
services/riprap-models/ The EO/forecast specialist HTTP service
|
| 92 |
+
|
| 93 |
+
scripts/
|
| 94 |
+
βββ probe_stones_fire.py Programmatic Stone-fire CI
|
| 95 |
+
βββ probe_addresses.py Canonical 5-address suite
|
| 96 |
+
βββ deploy_vllm_space.sh Deploy the L4 inference Space
|
| 97 |
+
βββ deploy_personal_space.sh Deploy the personal L4 mirror
|
| 98 |
+
βββ deploy_inference_space.sh Deploy the Ollama-backed inference Space
|
| 99 |
+
βββ β¦ Register builders, raster bakers, etc.
|
| 100 |
+
|
| 101 |
+
experiments/ Reproduction recipes for the three NYC fine-tunes
|
| 102 |
+
docs/ Architecture, methodology, deploy, emissions, runbooks
|
| 103 |
+
tests/ pytest suite (envelope + compare-shape tests)
|
| 104 |
+
```
|
| 105 |
+
|
| 106 |
+
## Style
|
| 107 |
+
|
| 108 |
+
- Python 3.12; `uv` for package management.
|
| 109 |
+
- LLM calls go through `app/llm.py` β never import `litellm` /
|
| 110 |
+
`ollama` directly from a specialist. The `chat()` shim wraps both
|
| 111 |
+
backends and the energy ledger reads off it.
|
| 112 |
+
- Remote ML calls go through `app/inference.py::_post`. Specialists
|
| 113 |
+
may try local fallback only when `inference.remote_enabled()` is
|
| 114 |
+
False; once a remote call has been attempted, return a clean
|
| 115 |
+
`{ok: False, skipped: ...}` on failure rather than crashing
|
| 116 |
+
through to local code paths that may not be installed.
|
| 117 |
+
- Every specialist emits one trace record per call with `step` /
|
| 118 |
+
`ok` / `elapsed_s` / `result` / `err` so the SSE stream and the
|
| 119 |
+
emissions tracker can reason about it.
|
| 120 |
+
|
| 121 |
+
## Reporting issues
|
| 122 |
+
|
| 123 |
+
GitHub issues at <https://github.com/msradam/riprap-nyc/issues>.
|
| 124 |
+
For hackathon-period demo issues during May 4β10 2026, the live
|
| 125 |
+
deploy at
|
| 126 |
+
<https://lablab-ai-amd-developer-hackathon-riprap-nyc.hf.space>
|
| 127 |
+
is the source of truth.
|
Dockerfile
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Riprap β Hugging Face Spaces deployment for the personal Space
|
| 2 |
+
# (msradam/riprap-nyc) on L4 hardware.
|
| 3 |
+
#
|
| 4 |
+
# Differences from the canonical Dockerfile:
|
| 5 |
+
#
|
| 6 |
+
# 1. L4 has 24 GB VRAM (vs 16 GB on T4 small), so we co-host the
|
| 7 |
+
# riprap-models service inside the same container instead of
|
| 8 |
+
# proxying to the AMD MI300X droplet. No external dependency.
|
| 9 |
+
#
|
| 10 |
+
# 2. We bake granite4.1:8b at *build* time. The build sandbox could
|
| 11 |
+
# not previously fit Granite + EO toolchain together; this Dockerfile
|
| 12 |
+
# keeps the EO install at runtime (entrypoint.l4.sh) and frees the
|
| 13 |
+
# sandbox budget for the 8B pull.
|
| 14 |
+
#
|
| 15 |
+
# 3. CUDA + ROCm-free torch β the inline riprap-models service uses
|
| 16 |
+
# the cu124 wheels installed via requirements.txt + the additional
|
| 17 |
+
# delta in services/riprap-models/requirements.txt.
|
| 18 |
+
#
|
| 19 |
+
# DO NOT push this image to the lablab Space β that one stays pointed
|
| 20 |
+
# at the MI300X droplet for AMD-judging continuity.
|
| 21 |
+
|
| 22 |
+
FROM nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04 AS base
|
| 23 |
+
|
| 24 |
+
ENV DEBIAN_FRONTEND=noninteractive
|
| 25 |
+
RUN apt-get update && apt-get install -y --no-install-recommends \
|
| 26 |
+
python3 python3-pip python3-venv python-is-python3 \
|
| 27 |
+
curl ca-certificates zstd procps git \
|
| 28 |
+
gdal-bin libgdal-dev libgeos-dev libproj-dev \
|
| 29 |
+
libgl1 libglib2.0-0 \
|
| 30 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 31 |
+
|
| 32 |
+
RUN useradd -m -u 1000 user
|
| 33 |
+
ENV HOME=/home/user \
|
| 34 |
+
PATH=/home/user/.local/bin:/usr/local/bin:/usr/bin:/bin \
|
| 35 |
+
PYTHONUNBUFFERED=1 \
|
| 36 |
+
HF_HOME=/home/user/.cache/huggingface \
|
| 37 |
+
OLLAMA_HOST=127.0.0.1:11434 \
|
| 38 |
+
OLLAMA_NUM_PARALLEL=1 \
|
| 39 |
+
OLLAMA_KEEP_ALIVE=24h \
|
| 40 |
+
OLLAMA_MAX_LOADED_MODELS=2 \
|
| 41 |
+
OLLAMA_FLASH_ATTENTION=1 \
|
| 42 |
+
OLLAMA_KV_CACHE_TYPE=q8_0 \
|
| 43 |
+
OLLAMA_DEBUG=1 \
|
| 44 |
+
OLLAMA_MODELS=/home/user/.ollama/models \
|
| 45 |
+
RIPRAP_OLLAMA_3B_TAG=granite4.1:8b \
|
| 46 |
+
RIPRAP_LLM_PRIMARY=ollama \
|
| 47 |
+
RIPRAP_LLM_BASE_URL=http://127.0.0.1:11434/v1 \
|
| 48 |
+
RIPRAP_ML_BACKEND=remote \
|
| 49 |
+
RIPRAP_ML_BASE_URL=http://127.0.0.1:7861
|
| 50 |
+
|
| 51 |
+
RUN curl -fsSL https://ollama.com/install.sh | sh
|
| 52 |
+
|
| 53 |
+
WORKDIR /home/user/app
|
| 54 |
+
|
| 55 |
+
# Web app deps (torch cu124 lands via sentence-transformers / etc.).
|
| 56 |
+
COPY --chown=user:user requirements.txt ./
|
| 57 |
+
RUN pip install --no-cache-dir --upgrade pip && \
|
| 58 |
+
pip install --no-cache-dir -r requirements.txt
|
| 59 |
+
|
| 60 |
+
# riprap-models delta deps. Use the existing requirements.txt at the
|
| 61 |
+
# *service* level, but skip requirements-full.txt β its ROCm-frozen
|
| 62 |
+
# torch pin would clobber the cu124 wheels installed above.
|
| 63 |
+
COPY --chown=user:user services/riprap-models/requirements.txt /tmp/req-models.txt
|
| 64 |
+
RUN pip install --no-cache-dir -r /tmp/req-models.txt
|
| 65 |
+
|
| 66 |
+
# Bake torchvision (CUDA 12.4 wheel) and peft at build time. The
|
| 67 |
+
# canonical entrypoint.sh runtime-installs torchvision via the EO
|
| 68 |
+
# toolchain path because the canonical CPU Space's build sandbox is
|
| 69 |
+
# too tight; L4 builds have more room, and a properly matched
|
| 70 |
+
# torchvision avoids the `torchvision::nms does not exist` runtime
|
| 71 |
+
# error the canonical setup hits. peft is required by the riprap-
|
| 72 |
+
# models service for the TerraMind LoRA inference path.
|
| 73 |
+
RUN pip install --no-cache-dir \
|
| 74 |
+
--index-url https://download.pytorch.org/whl/cu124 \
|
| 75 |
+
torchvision \
|
| 76 |
+
&& pip install --no-cache-dir peft==0.18.1
|
| 77 |
+
|
| 78 |
+
# Bake Granite 4.1 weights into the image (EO toolchain is installed
|
| 79 |
+
# at runtime β see entrypoint.l4.sh β to keep the build sandbox under
|
| 80 |
+
# its disk threshold).
|
| 81 |
+
RUN mkdir -p $OLLAMA_MODELS && \
|
| 82 |
+
ollama serve & \
|
| 83 |
+
OPID=$! && \
|
| 84 |
+
for i in $(seq 1 30); do curl -sf http://127.0.0.1:11434/ > /dev/null && break; sleep 1; done && \
|
| 85 |
+
ollama pull granite4.1:8b && \
|
| 86 |
+
kill $OPID 2>/dev/null || true && \
|
| 87 |
+
sleep 2
|
| 88 |
+
|
| 89 |
+
# App code, fixtures, and inline model service.
|
| 90 |
+
COPY --chown=user:user app/ ./app/
|
| 91 |
+
COPY --chown=user:user web/ ./web/
|
| 92 |
+
COPY --chown=user:user scripts/ ./scripts/
|
| 93 |
+
COPY --chown=user:user data/ ./data/
|
| 94 |
+
COPY --chown=user:user corpus/ ./corpus/
|
| 95 |
+
COPY --chown=user:user services/riprap-models/main.py ./riprap_models.py
|
| 96 |
+
COPY --chown=user:user agent.py riprap.py ./
|
| 97 |
+
COPY --chown=user:user entrypoint.sh ./entrypoint.sh
|
| 98 |
+
RUN chmod +x ./entrypoint.sh
|
| 99 |
+
|
| 100 |
+
RUN chown -R user:user /home/user
|
| 101 |
+
USER user
|
| 102 |
+
|
| 103 |
+
EXPOSE 7860
|
| 104 |
+
CMD ["./entrypoint.sh"]
|
README.md
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
title: Riprap NYC (Personal Mirror, L4)
|
| 3 |
+
emoji: π
|
| 4 |
+
colorFrom: blue
|
| 5 |
+
colorTo: indigo
|
| 6 |
+
sdk: docker
|
| 7 |
+
pinned: false
|
| 8 |
+
short_description: NYC flood-exposure briefings on L4 (self-contained).
|
| 9 |
+
---
|
| 10 |
+
|
| 11 |
+
# Riprap β NYC flood-exposure briefings (L4 self-contained mirror)
|
| 12 |
+
|
| 13 |
+
This Space is a self-contained mirror of
|
| 14 |
+
[`github.com/msradam/riprap-nyc`](https://github.com/msradam/riprap-nyc).
|
| 15 |
+
|
| 16 |
+
It runs on a single L4 GPU and co-hosts everything in one container:
|
| 17 |
+
Granite 4.1 8B (via Ollama), Prithvi-EO 2.0 NYC-Pluvial, TerraMind
|
| 18 |
+
LULC + Buildings LoRAs, and Granite TTM r2 β no external droplet
|
| 19 |
+
dependency. Sleeps on idle; first request after sleep takes ~45β60 s
|
| 20 |
+
to wake.
|
| 21 |
+
|
| 22 |
+
The hackathon submission Space (CPU UI, droplet proxy) lives at
|
| 23 |
+
[`AMD-hackathon/riprap-nyc`](https://lablab-ai-amd-developer-hackathon-riprap-nyc.hf.space).
|
| 24 |
+
|
| 25 |
+
Apache 2.0. See the GitHub repo for full source, architecture
|
| 26 |
+
deep-dive, methodology, and licence map.
|
SECURITY.md
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Security policy
|
| 2 |
+
|
| 3 |
+
## Reporting a vulnerability
|
| 4 |
+
|
| 5 |
+
If you find a security issue in Riprap, please report it privately so
|
| 6 |
+
it can be triaged before disclosure.
|
| 7 |
+
|
| 8 |
+
- Email: **msrahmanadam@gmail.com** (subject prefix: `[riprap-security]`)
|
| 9 |
+
- Or open a [GitHub Security Advisory](https://github.com/msradam/riprap-nyc/security/advisories/new)
|
| 10 |
+
on this repository.
|
| 11 |
+
|
| 12 |
+
Please do not file a public GitHub issue for security reports.
|
| 13 |
+
|
| 14 |
+
We aim to acknowledge reports within 72 hours and to ship a fix or a
|
| 15 |
+
mitigation plan within two weeks of triage. If the report concerns a
|
| 16 |
+
vulnerability in an upstream model or service Riprap depends on
|
| 17 |
+
(IBM Granite, vLLM, Hugging Face Spaces, NYC Open Data endpoints), we
|
| 18 |
+
will help coordinate disclosure with the upstream maintainer.
|
| 19 |
+
|
| 20 |
+
## Threat-surface notes
|
| 21 |
+
|
| 22 |
+
Riprap is a citation-grounded synthesis layer over public-record
|
| 23 |
+
data. By design, the runtime:
|
| 24 |
+
|
| 25 |
+
- contacts only **public-record APIs** (NYC Open Data, FloodNet,
|
| 26 |
+
USGS, NOAA, NWS, NYS DOH, MTA, NYCHA, NYC DOE, OpenStreetMap /
|
| 27 |
+
Nominatim) and the configured inference Spaces;
|
| 28 |
+
- does **not** authenticate against user accounts or store
|
| 29 |
+
user-identifying data β the address bar is the only input;
|
| 30 |
+
- runs the SvelteKit UI as a static SPA over a FastAPI backend
|
| 31 |
+
with no persistent database.
|
| 32 |
+
|
| 33 |
+
The vulnerability surface is therefore small. Plausible categories
|
| 34 |
+
worth a report:
|
| 35 |
+
|
| 36 |
+
- Prompt-injection paths via document content that escape the
|
| 37 |
+
Mellea grounding loop and surface unverifiable claims as cited.
|
| 38 |
+
- SSRF / abuse via crafted address strings that drive backend
|
| 39 |
+
HTTP calls to unintended hosts.
|
| 40 |
+
- Token leakage in proxy headers or SSE streams
|
| 41 |
+
(`inference-vllm/proxy.py`, `web/main.py`).
|
| 42 |
+
- Denial-of-service patterns that exceed the hosted Space's
|
| 43 |
+
resource budget.
|
| 44 |
+
- Supply-chain issues in pinned deps (`requirements.txt`,
|
| 45 |
+
`web/sveltekit/package.json`).
|
| 46 |
+
|
| 47 |
+
## Out of scope
|
| 48 |
+
|
| 49 |
+
- Self-hosted deployments running with custom configuration or
|
| 50 |
+
custom datasets β please file those as regular bugs.
|
| 51 |
+
- Findings that require physical or local-network access to a
|
| 52 |
+
user's machine.
|
| 53 |
+
- Issues in the lablab.ai or Hugging Face Spaces hosting platforms
|
| 54 |
+
themselves; please report those upstream.
|
agent.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Riprap agent CLI β address β cited briefing via the Burr FSM.
|
| 2 |
+
|
| 3 |
+
Usage:
|
| 4 |
+
python agent.py "180 Beach 35 St, Queens"
|
| 5 |
+
python agent.py "280 Broome St, Manhattan" --json
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import argparse
|
| 10 |
+
import json
|
| 11 |
+
import sys
|
| 12 |
+
import warnings
|
| 13 |
+
|
| 14 |
+
warnings.filterwarnings("ignore")
|
| 15 |
+
|
| 16 |
+
from app.fsm import run # noqa: E402
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def main() -> int:
|
| 20 |
+
ap = argparse.ArgumentParser()
|
| 21 |
+
ap.add_argument("query", help="NYC address or natural-language location")
|
| 22 |
+
ap.add_argument("--json", action="store_true", help="emit full JSON state")
|
| 23 |
+
args = ap.parse_args()
|
| 24 |
+
|
| 25 |
+
print(f"\n query: {args.query}", file=sys.stderr)
|
| 26 |
+
print(" running FSM... (Granite 4.1 + open data, all local)\n", file=sys.stderr)
|
| 27 |
+
|
| 28 |
+
result = run(args.query)
|
| 29 |
+
|
| 30 |
+
if args.json:
|
| 31 |
+
print(json.dumps(result, indent=2, default=str))
|
| 32 |
+
return 0
|
| 33 |
+
|
| 34 |
+
print("βββ trace " + "β" * 56)
|
| 35 |
+
for step in result["trace"]:
|
| 36 |
+
ok = "β" if step["ok"] else "β"
|
| 37 |
+
line = f" {ok} {step['step']:22s} {step.get('elapsed_s', 0):>5.2f}s"
|
| 38 |
+
if step.get("result"):
|
| 39 |
+
line += " " + json.dumps(step["result"], default=str)
|
| 40 |
+
elif step.get("err"):
|
| 41 |
+
line += " ERR: " + step["err"]
|
| 42 |
+
print(line)
|
| 43 |
+
|
| 44 |
+
print("\nβββ cited report " + "β" * 49)
|
| 45 |
+
print()
|
| 46 |
+
print(result["paragraph"])
|
| 47 |
+
print()
|
| 48 |
+
return 0
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
if __name__ == "__main__":
|
| 52 |
+
sys.exit(main())
|
app/__init__.py
ADDED
|
File without changes
|
app/areas/__init__.py
ADDED
|
File without changes
|
app/areas/nta.py
ADDED
|
@@ -0,0 +1,224 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""NYC Neighborhood Tabulation Area (NTA 2020) resolver.
|
| 2 |
+
|
| 3 |
+
NTAs are NYC Department of City Planning's official neighborhood unit:
|
| 4 |
+
~262 polygons covering all 5 boroughs, including some park / airport
|
| 5 |
+
slivers. They are the canonical "neighborhood" unit for NYC civic data.
|
| 6 |
+
|
| 7 |
+
This module provides:
|
| 8 |
+
- load() β GeoDataFrame with all NTAs (cached)
|
| 9 |
+
- resolve(name) β list of matching NTAs by fuzzy name match, or by borough
|
| 10 |
+
- by_code(code) β exact lookup
|
| 11 |
+
- polygon_for(code) β shapely Polygon in EPSG:4326
|
| 12 |
+
"""
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import re
|
| 16 |
+
from functools import lru_cache
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
from typing import Any
|
| 19 |
+
|
| 20 |
+
import geopandas as gpd
|
| 21 |
+
from shapely.geometry import Polygon
|
| 22 |
+
|
| 23 |
+
DATA_PATH = Path(__file__).resolve().parents[2] / "data" / "nyc_ntas_2020.geojson"
|
| 24 |
+
|
| 25 |
+
# Common alias map: user-typed strings β canonical NTA names. We don't need to
|
| 26 |
+
# be exhaustive here; the fuzzy matcher catches most cases. This handles the
|
| 27 |
+
# few hard ones where the official NTA name differs from local usage.
|
| 28 |
+
ALIASES = {
|
| 29 |
+
"the rockaways": "Rockaway Beach-Arverne-Edgemere",
|
| 30 |
+
"rockaway": "Rockaway Beach-Arverne-Edgemere",
|
| 31 |
+
"brighton": "Brighton Beach",
|
| 32 |
+
"lower east side": "Lower East Side",
|
| 33 |
+
"les": "Lower East Side",
|
| 34 |
+
"soho": "SoHo-Little Italy-Hudson Square",
|
| 35 |
+
"tribeca": "Tribeca-Civic Center",
|
| 36 |
+
"fidi": "Financial District-Battery Park City",
|
| 37 |
+
"downtown brooklyn":"Downtown Brooklyn-DUMBO-Boerum Hill",
|
| 38 |
+
"dumbo": "Downtown Brooklyn-DUMBO-Boerum Hill",
|
| 39 |
+
"park slope": "Park Slope",
|
| 40 |
+
"carroll gardens": "Carroll Gardens-Cobble Hill-Gowanus-Red Hook",
|
| 41 |
+
"red hook": "Carroll Gardens-Cobble Hill-Gowanus-Red Hook",
|
| 42 |
+
"gowanus": "Carroll Gardens-Cobble Hill-Gowanus-Red Hook",
|
| 43 |
+
"hollis": "Queens Village-Hollis-Bellerose",
|
| 44 |
+
"long island city": "Hunters Point-Sunnyside-West Maspeth",
|
| 45 |
+
"lic": "Hunters Point-Sunnyside-West Maspeth",
|
| 46 |
+
"astoria": "Astoria (Central)",
|
| 47 |
+
"flushing": "Flushing-Willets Point",
|
| 48 |
+
"harlem": "Central Harlem (North)",
|
| 49 |
+
"east harlem": "East Harlem (North)",
|
| 50 |
+
"washington heights":"Washington Heights (North)",
|
| 51 |
+
"midtown": "Midtown South-Flatiron-Union Square",
|
| 52 |
+
"upper east side": "Upper East Side-Carnegie Hill",
|
| 53 |
+
"ues": "Upper East Side-Carnegie Hill",
|
| 54 |
+
"upper west side": "Upper West Side-Lincoln Square",
|
| 55 |
+
"uws": "Upper West Side-Lincoln Square",
|
| 56 |
+
"coney island": "Coney Island-Sea Gate",
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
BOROUGH_NORMALIZE = {
|
| 60 |
+
"manhattan": "Manhattan", "mn": "Manhattan",
|
| 61 |
+
"brooklyn": "Brooklyn", "bk": "Brooklyn", "kings": "Brooklyn",
|
| 62 |
+
"queens": "Queens", "qn": "Queens",
|
| 63 |
+
"bronx": "Bronx", "the bronx": "Bronx", "bx": "Bronx",
|
| 64 |
+
"staten island": "Staten Island", "si": "Staten Island", "richmond": "Staten Island",
|
| 65 |
+
}
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def _normalize(s: str) -> str:
|
| 69 |
+
return re.sub(r"[^a-z]+", "", (s or "").lower())
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
@lru_cache(maxsize=1)
|
| 73 |
+
def load() -> gpd.GeoDataFrame:
|
| 74 |
+
"""Load the NTA 2020 GeoJSON; coerce CRS to EPSG:4326. Cached."""
|
| 75 |
+
g = gpd.read_file(DATA_PATH)
|
| 76 |
+
if g.crs is None or g.crs.to_string() != "EPSG:4326":
|
| 77 |
+
g = g.to_crs("EPSG:4326")
|
| 78 |
+
return g
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def by_code(code: str) -> dict | None:
|
| 82 |
+
g = load()
|
| 83 |
+
hit = g[g["nta2020"] == code]
|
| 84 |
+
if hit.empty:
|
| 85 |
+
return None
|
| 86 |
+
return _row_to_dict(hit.iloc[0])
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def _row_to_dict(row) -> dict:
|
| 90 |
+
return {
|
| 91 |
+
"nta_code": row["nta2020"],
|
| 92 |
+
"nta_name": row["ntaname"],
|
| 93 |
+
"borough": row["boroname"],
|
| 94 |
+
"cdta": row.get("cdtaname"),
|
| 95 |
+
"geometry": row["geometry"],
|
| 96 |
+
}
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def borough_match(query: str) -> str | None:
|
| 100 |
+
"""If query matches a borough name (or common abbreviation), return the
|
| 101 |
+
canonical name. Otherwise return None."""
|
| 102 |
+
q = query.strip().lower()
|
| 103 |
+
return BOROUGH_NORMALIZE.get(q)
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def resolve(query: str) -> list[dict[str, Any]]:
|
| 107 |
+
"""Resolve a free-text query to NTA(s).
|
| 108 |
+
|
| 109 |
+
Strategy (in priority order):
|
| 110 |
+
1. Borough match β all NTAs in borough.
|
| 111 |
+
2. Alias map β exact NTA name match.
|
| 112 |
+
3. Case-insensitive EXACT name match (so 'Kew Gardens' wins over
|
| 113 |
+
'Kew Gardens Hills' when both exist).
|
| 114 |
+
4. Substring match on normalized NTA name. When multiple match,
|
| 115 |
+
prefer the one whose normalized name length is closest to the
|
| 116 |
+
query β avoids 'Kew Gardens' resolving to 'Kew Gardens Hills'.
|
| 117 |
+
5. CDTA-name substring fallback.
|
| 118 |
+
"""
|
| 119 |
+
g = load()
|
| 120 |
+
q = (query or "").strip()
|
| 121 |
+
if not q:
|
| 122 |
+
return []
|
| 123 |
+
boro = borough_match(q)
|
| 124 |
+
if boro:
|
| 125 |
+
hits = g[g["boroname"] == boro]
|
| 126 |
+
return [_row_to_dict(r) for _, r in hits.iterrows()]
|
| 127 |
+
|
| 128 |
+
alias = ALIASES.get(q.lower())
|
| 129 |
+
if alias:
|
| 130 |
+
hits = g[g["ntaname"] == alias]
|
| 131 |
+
if not hits.empty:
|
| 132 |
+
return [_row_to_dict(r) for _, r in hits.iterrows()]
|
| 133 |
+
|
| 134 |
+
# Exact (case-insensitive) β preferred over substring
|
| 135 |
+
name_lower = g["ntaname"].fillna("").str.lower()
|
| 136 |
+
exact = g[name_lower == q.lower()]
|
| 137 |
+
if not exact.empty:
|
| 138 |
+
return [_row_to_dict(r) for _, r in exact.iterrows()]
|
| 139 |
+
|
| 140 |
+
qn = _normalize(q)
|
| 141 |
+
if not qn:
|
| 142 |
+
return []
|
| 143 |
+
name_norm = g["ntaname"].fillna("").map(_normalize)
|
| 144 |
+
contains = g[name_norm.str.contains(qn, na=False)].copy()
|
| 145 |
+
if not contains.empty:
|
| 146 |
+
contains["_diff"] = contains["ntaname"].fillna("").map(
|
| 147 |
+
lambda s: abs(len(_normalize(s)) - len(qn))
|
| 148 |
+
)
|
| 149 |
+
contains = contains.sort_values("_diff")
|
| 150 |
+
return [_row_to_dict(r) for _, r in contains.iterrows()]
|
| 151 |
+
|
| 152 |
+
cdta_norm = g["cdtaname"].fillna("").map(_normalize)
|
| 153 |
+
contains = g[cdta_norm.str.contains(qn, na=False)]
|
| 154 |
+
if not contains.empty:
|
| 155 |
+
return [_row_to_dict(r) for _, r in contains.iterrows()]
|
| 156 |
+
|
| 157 |
+
return []
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def polygon_for(code: str) -> Polygon | None:
|
| 161 |
+
hit = by_code(code)
|
| 162 |
+
return hit["geometry"] if hit else None
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
def resolve_from_text(text: str) -> list[dict[str, Any]]: # TODO(cleanup): cc-grade-D (25)
|
| 166 |
+
"""Scan free-text (e.g. a full natural-language query) for any known NTA
|
| 167 |
+
name, alias, or borough. Returns the first match. This is the fallback
|
| 168 |
+
when the planner failed to extract a clean target.
|
| 169 |
+
|
| 170 |
+
Strategy: walk ALIASES first (cheap), then iterate NTA names and look
|
| 171 |
+
for the longest match contained in the text. We prefer the longest
|
| 172 |
+
match so 'Carroll Gardens' wins over 'Gardens'.
|
| 173 |
+
"""
|
| 174 |
+
t = (text or "").lower()
|
| 175 |
+
if not t:
|
| 176 |
+
return []
|
| 177 |
+
# Boroughs first (whole-word-ish β avoid false hits inside "queensland" etc.)
|
| 178 |
+
for boro_key, canon in BOROUGH_NORMALIZE.items():
|
| 179 |
+
if f" {boro_key} " in f" {t} " or t.startswith(boro_key + " ") or t.endswith(" " + boro_key):
|
| 180 |
+
hits = resolve(canon)
|
| 181 |
+
if hits:
|
| 182 |
+
return hits
|
| 183 |
+
# Alias keys, longest first
|
| 184 |
+
for key in sorted(ALIASES.keys(), key=len, reverse=True):
|
| 185 |
+
if key in t:
|
| 186 |
+
hits = resolve(key)
|
| 187 |
+
if hits:
|
| 188 |
+
return hits
|
| 189 |
+
# NTA names. Order: longest first so multi-word names match before
|
| 190 |
+
# shorter substrings, AND preferring the WORD-BOUNDARY match so
|
| 191 |
+
# "Kew Gardens" in the query doesn't collide with "Kew Gardens Hills"
|
| 192 |
+
# (the latter is longer; without word-boundary checking it'd match
|
| 193 |
+
# nothing, but with substring-in-text it'd match if the query ever
|
| 194 |
+
# contained the longer phrase). Caller picks the closest-length match.
|
| 195 |
+
g = load()
|
| 196 |
+
names = sorted(set(g["ntaname"].dropna().str.lower().tolist()), key=len, reverse=True)
|
| 197 |
+
matches = []
|
| 198 |
+
for name in names:
|
| 199 |
+
if not name or len(name) < 4:
|
| 200 |
+
continue
|
| 201 |
+
# Word-boundary-ish check: name must appear bounded by start/end or
|
| 202 |
+
# whitespace/punct (so "kew gardens hills" matches but "kew gardens"
|
| 203 |
+
# alone doesn't trigger "kew gardens hills" because of the trailing
|
| 204 |
+
# space requirement).
|
| 205 |
+
padded_t = f" {t} "
|
| 206 |
+
if f" {name} " in padded_t or f" {name}." in padded_t or f" {name}," in padded_t or f" {name}?" in padded_t:
|
| 207 |
+
matches.append(name)
|
| 208 |
+
if matches:
|
| 209 |
+
# Prefer the longest word-boundary match β most specific.
|
| 210 |
+
best = sorted(matches, key=len, reverse=True)[0]
|
| 211 |
+
hits = resolve(best)
|
| 212 |
+
if hits:
|
| 213 |
+
return hits
|
| 214 |
+
# Fallback: any substring (no boundary). Less precise, but catches
|
| 215 |
+
# casual queries like "show me red hook" where "red hook" is a
|
| 216 |
+
# neighborhood-name fragment within a longer NTA name.
|
| 217 |
+
for name in names:
|
| 218 |
+
if not name or len(name) < 4:
|
| 219 |
+
continue
|
| 220 |
+
if name in t:
|
| 221 |
+
hits = resolve(name)
|
| 222 |
+
if hits:
|
| 223 |
+
return hits
|
| 224 |
+
return []
|
app/assets/__init__.py
ADDED
|
File without changes
|
app/assets/mta_entrances.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""MTA Subway Entrances and Exits (NY OpenData i9wp-a4ja).
|
| 2 |
+
|
| 3 |
+
~1,900 subway entrances city-wide. The MTA Climate Resilience Roadmap
|
| 4 |
+
(Oct 2025) names ~1,500 of these as priorities for sealing β this is
|
| 5 |
+
exactly the asset class our RAG corpus has the most to say about, and
|
| 6 |
+
exactly the audience (MTA capital planners, transit advocacy) the
|
| 7 |
+
register is built for.
|
| 8 |
+
"""
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
|
| 13 |
+
import geopandas as gpd
|
| 14 |
+
import httpx
|
| 15 |
+
|
| 16 |
+
from app.spatial import DATA, NYC_CRS
|
| 17 |
+
|
| 18 |
+
URL = "https://data.ny.gov/api/geospatial/i9wp-a4ja?method=export&format=GeoJSON"
|
| 19 |
+
LOCAL = DATA / "mta_entrances.geojson"
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def _ensure_fixture() -> Path:
|
| 23 |
+
if LOCAL.exists():
|
| 24 |
+
return LOCAL
|
| 25 |
+
print("downloading MTA Subway Entrances (one-time)...", flush=True)
|
| 26 |
+
r = httpx.get(URL, timeout=60)
|
| 27 |
+
r.raise_for_status()
|
| 28 |
+
LOCAL.write_text(r.text)
|
| 29 |
+
return LOCAL
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def load() -> gpd.GeoDataFrame:
|
| 33 |
+
_ensure_fixture()
|
| 34 |
+
g = gpd.read_file(LOCAL)
|
| 35 |
+
if g.crs is None:
|
| 36 |
+
g.set_crs("EPSG:4326", inplace=True)
|
| 37 |
+
g = g.to_crs(NYC_CRS)
|
| 38 |
+
rename_map = {
|
| 39 |
+
"stop_name": "name",
|
| 40 |
+
"constrained_floor_to_floor_height": None,
|
| 41 |
+
"borough": "borough",
|
| 42 |
+
"entrance_type": "entrance_type",
|
| 43 |
+
"ada": "ada",
|
| 44 |
+
"north_south_street": "ns_street",
|
| 45 |
+
"east_west_street": "ew_street",
|
| 46 |
+
"corner": "corner",
|
| 47 |
+
}
|
| 48 |
+
for k, v in rename_map.items():
|
| 49 |
+
if v and k in g.columns and k != v:
|
| 50 |
+
g = g.rename(columns={k: v})
|
| 51 |
+
|
| 52 |
+
# build a usable address-style label
|
| 53 |
+
def label(row):
|
| 54 |
+
nm = (row.get("name") or "").strip()
|
| 55 |
+
ns = (row.get("ns_street") or "").strip()
|
| 56 |
+
ew = (row.get("ew_street") or "").strip()
|
| 57 |
+
cn = (row.get("corner") or "").strip()
|
| 58 |
+
bits = [nm]
|
| 59 |
+
cross = " & ".join(b for b in [ns, ew] if b)
|
| 60 |
+
if cross: bits.append(cross)
|
| 61 |
+
if cn: bits.append(f"({cn})")
|
| 62 |
+
return ", ".join([b for b in bits if b])
|
| 63 |
+
|
| 64 |
+
g["address"] = g.apply(label, axis=1)
|
| 65 |
+
if "borough" in g.columns:
|
| 66 |
+
boro_map = {"M": "Manhattan", "Bk": "Brooklyn", "B": "Brooklyn",
|
| 67 |
+
"Q": "Queens", "Bx": "Bronx", "SI": "Staten Island"}
|
| 68 |
+
g["borough"] = g["borough"].astype(str).map(lambda v: boro_map.get(v, v.title()))
|
| 69 |
+
|
| 70 |
+
keep = [c for c in ["name", "address", "borough", "entrance_type",
|
| 71 |
+
"ada", "ns_street", "ew_street", "corner", "geometry"]
|
| 72 |
+
if c in g.columns]
|
| 73 |
+
return g[keep].copy()
|
app/assets/nycha.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""NYCHA Developments (NYC OpenData phvi-damg).
|
| 2 |
+
|
| 3 |
+
326 public-housing developments across NYC. Used as an asset class for
|
| 4 |
+
the bulk-mode register; the parent rationale for surfacing this layer
|
| 5 |
+
is that NYCHA was hit hard by Sandy and remains a published Tier-1
|
| 6 |
+
flood-resilience priority in the city's Hazard Mitigation Plan.
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import geopandas as gpd
|
| 11 |
+
|
| 12 |
+
from app.spatial import DATA, load_layer
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def load() -> gpd.GeoDataFrame:
|
| 16 |
+
g = load_layer(DATA / "nycha.geojson")
|
| 17 |
+
# NYCHA developments come back as polygons; the FSM expects point
|
| 18 |
+
# geometry for spatial joins. Use centroid.
|
| 19 |
+
g = g.copy()
|
| 20 |
+
g["geometry"] = g.geometry.centroid
|
| 21 |
+
|
| 22 |
+
# NYCHA Developments has only `developmen` (truncated label), tds_num, borough.
|
| 23 |
+
g = g.rename(columns={"developmen": "name"})
|
| 24 |
+
g["address"] = g["name"] # the field doubles as both
|
| 25 |
+
g["borough"] = g["borough"].str.title() # "BRONX" -> "Bronx" to match Riprap convention
|
| 26 |
+
|
| 27 |
+
keep = [c for c in ["name", "address", "borough", "tds_num", "geometry"] if c in g.columns]
|
| 28 |
+
return g[keep].copy()
|
app/assets/schools.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""NYC DOE School Point Locations (Socrata a3nt-yts4)."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import geopandas as gpd
|
| 5 |
+
|
| 6 |
+
from app.spatial import DATA, load_layer
|
| 7 |
+
|
| 8 |
+
BORO = {"1": "Manhattan", "2": "Bronx", "3": "Brooklyn", "4": "Queens", "5": "Staten Island"}
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def load() -> gpd.GeoDataFrame:
|
| 12 |
+
g = load_layer(DATA / "schools.geojson")
|
| 13 |
+
g = g.rename(columns={
|
| 14 |
+
"loc_code": "loc_code",
|
| 15 |
+
"loc_name": "name",
|
| 16 |
+
"address": "address",
|
| 17 |
+
"bbl": "bbl",
|
| 18 |
+
"bin": "bin",
|
| 19 |
+
"boronum": "boro_num",
|
| 20 |
+
"geodistric": "geo_district",
|
| 21 |
+
"adimindist": "admin_district",
|
| 22 |
+
})
|
| 23 |
+
g["borough"] = g["boro_num"].astype(str).map(BORO)
|
| 24 |
+
g["bbl"] = g["bbl"].astype(str).str.replace(r"\.0$", "", regex=True)
|
| 25 |
+
keep = ["loc_code", "name", "address", "borough", "bbl", "bin",
|
| 26 |
+
"geo_district", "admin_district", "geometry"]
|
| 27 |
+
return g[keep].copy()
|
app/context/__init__.py
ADDED
|
File without changes
|
app/context/_polygonize.py
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Vectorize a uint8 prediction raster (binary mask or class index)
|
| 2 |
+
into an EPSG:4326 GeoJSON FeatureCollection so the frontend can paint
|
| 3 |
+
it on the MapLibre map.
|
| 4 |
+
|
| 5 |
+
The droplet's `/v1/prithvi-pluvial` and `/v1/terramind` routes return
|
| 6 |
+
their predictions as base64-encoded uint8 with a shape and (where
|
| 7 |
+
relevant) a class-label list. This module reconstructs the affine
|
| 8 |
+
transform from the chip's geographic bounds (which the HF Space
|
| 9 |
+
already knows) and walks `rasterio.features.shapes` to build polygons
|
| 10 |
+
in the chip's native CRS, then reprojects to WGS84 for the map.
|
| 11 |
+
|
| 12 |
+
Best-effort: any failure returns an empty FeatureCollection rather
|
| 13 |
+
than raising into the caller's path. The map layer is decorative β
|
| 14 |
+
the briefing is the deliverable.
|
| 15 |
+
"""
|
| 16 |
+
from __future__ import annotations
|
| 17 |
+
|
| 18 |
+
import base64
|
| 19 |
+
import logging
|
| 20 |
+
|
| 21 |
+
log = logging.getLogger("riprap.polygonize")
|
| 22 |
+
|
| 23 |
+
EMPTY: dict = {"type": "FeatureCollection", "features": []}
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def _decode_pred(pred_b64: str, pred_shape: list[int]):
|
| 27 |
+
"""Inverse of the droplet's `base64(pred.tobytes())`. Returns a
|
| 28 |
+
uint8 numpy array of shape `pred_shape`, or None on decode error."""
|
| 29 |
+
try:
|
| 30 |
+
import numpy as np
|
| 31 |
+
raw = base64.b64decode(pred_b64)
|
| 32 |
+
return np.frombuffer(raw, dtype="uint8").reshape(pred_shape)
|
| 33 |
+
except Exception:
|
| 34 |
+
log.exception("polygonize: pred decode failed")
|
| 35 |
+
return None
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def polygonize_class_raster(
|
| 39 |
+
pred_b64: str,
|
| 40 |
+
pred_shape: list[int],
|
| 41 |
+
class_labels: list[str] | None,
|
| 42 |
+
bounds_4326: tuple[float, float, float, float],
|
| 43 |
+
*,
|
| 44 |
+
drop_classes: tuple[int, ...] = (0,),
|
| 45 |
+
simplify_tolerance: float = 0.0,
|
| 46 |
+
) -> dict:
|
| 47 |
+
"""Vectorize a categorical prediction raster (one integer class per
|
| 48 |
+
pixel) into a FeatureCollection with one Feature per connected
|
| 49 |
+
polygon. `bounds_4326` is `(minlon, minlat, maxlon, maxlat)` of the
|
| 50 |
+
chip; the raster is assumed to span those bounds at uniform
|
| 51 |
+
pixel size. Each feature carries `class_idx` and `class_label`
|
| 52 |
+
so the frontend can color by class.
|
| 53 |
+
|
| 54 |
+
`drop_classes`: skip pixels matching these class indices (default
|
| 55 |
+
drops 0 = "Background" / "outside" / etc).
|
| 56 |
+
"""
|
| 57 |
+
pred = _decode_pred(pred_b64, pred_shape)
|
| 58 |
+
if pred is None:
|
| 59 |
+
return EMPTY
|
| 60 |
+
try:
|
| 61 |
+
from rasterio.features import shapes
|
| 62 |
+
from rasterio.transform import from_bounds
|
| 63 |
+
from shapely.geometry import shape
|
| 64 |
+
h, w = pred.shape
|
| 65 |
+
minlon, minlat, maxlon, maxlat = bounds_4326
|
| 66 |
+
# The chip is in EPSG:4326 for our use β Sentinel-2 chips are
|
| 67 |
+
# natively in their UTM zone, but we can polygonize against the
|
| 68 |
+
# WGS84 extent because the inference chip is a small bbox where
|
| 69 |
+
# the pixel-grid β lat/lon mapping is locally affine (sub-pixel
|
| 70 |
+
# error at NYC scale).
|
| 71 |
+
transform = from_bounds(minlon, minlat, maxlon, maxlat, w, h)
|
| 72 |
+
feats = []
|
| 73 |
+
for geom, value in shapes(pred, mask=pred > 0, transform=transform):
|
| 74 |
+
v = int(value)
|
| 75 |
+
if v in drop_classes:
|
| 76 |
+
continue
|
| 77 |
+
label = (class_labels[v]
|
| 78 |
+
if class_labels and 0 <= v < len(class_labels)
|
| 79 |
+
else f"class_{v}")
|
| 80 |
+
poly = shape(geom)
|
| 81 |
+
if simplify_tolerance > 0:
|
| 82 |
+
poly = poly.simplify(simplify_tolerance, preserve_topology=True)
|
| 83 |
+
if poly.is_empty:
|
| 84 |
+
continue
|
| 85 |
+
feats.append({
|
| 86 |
+
"type": "Feature",
|
| 87 |
+
"geometry": poly.__geo_interface__,
|
| 88 |
+
"properties": {
|
| 89 |
+
"class_idx": v,
|
| 90 |
+
"class_label": label,
|
| 91 |
+
"fill_color": _PALETTE.get(label.lower(), _DEFAULT_FILL),
|
| 92 |
+
},
|
| 93 |
+
})
|
| 94 |
+
return {"type": "FeatureCollection", "features": feats}
|
| 95 |
+
except Exception:
|
| 96 |
+
log.exception("polygonize: class raster vectorisation failed")
|
| 97 |
+
return EMPTY
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def polygonize_binary_mask(
|
| 101 |
+
pred_b64: str,
|
| 102 |
+
pred_shape: list[int],
|
| 103 |
+
bounds_4326: tuple[float, float, float, float],
|
| 104 |
+
*,
|
| 105 |
+
label: str = "water",
|
| 106 |
+
fill_color: str = "#4A90E2",
|
| 107 |
+
simplify_tolerance: float = 0.0,
|
| 108 |
+
) -> dict:
|
| 109 |
+
"""Vectorize a binary prediction raster (e.g. Prithvi water mask;
|
| 110 |
+
1 = water, 0 = not). Returns one Feature per connected positive
|
| 111 |
+
region. Use this for prithvi_eo_live and the buildings LoRA."""
|
| 112 |
+
pred = _decode_pred(pred_b64, pred_shape)
|
| 113 |
+
if pred is None:
|
| 114 |
+
return EMPTY
|
| 115 |
+
try:
|
| 116 |
+
from rasterio.features import shapes
|
| 117 |
+
from rasterio.transform import from_bounds
|
| 118 |
+
from shapely.geometry import shape
|
| 119 |
+
h, w = pred.shape
|
| 120 |
+
minlon, minlat, maxlon, maxlat = bounds_4326
|
| 121 |
+
transform = from_bounds(minlon, minlat, maxlon, maxlat, w, h)
|
| 122 |
+
feats = []
|
| 123 |
+
for geom, _value in shapes(pred, mask=pred > 0, transform=transform):
|
| 124 |
+
poly = shape(geom)
|
| 125 |
+
if simplify_tolerance > 0:
|
| 126 |
+
poly = poly.simplify(simplify_tolerance, preserve_topology=True)
|
| 127 |
+
if poly.is_empty:
|
| 128 |
+
continue
|
| 129 |
+
feats.append({
|
| 130 |
+
"type": "Feature",
|
| 131 |
+
"geometry": poly.__geo_interface__,
|
| 132 |
+
"properties": {
|
| 133 |
+
"class_label": label,
|
| 134 |
+
"fill_color": fill_color,
|
| 135 |
+
},
|
| 136 |
+
})
|
| 137 |
+
return {"type": "FeatureCollection", "features": feats}
|
| 138 |
+
except Exception:
|
| 139 |
+
log.exception("polygonize: binary mask vectorisation failed")
|
| 140 |
+
return EMPTY
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
# Lightweight palette used by the LULC + buildings layers. Frontend
|
| 144 |
+
# may override via `fill_color` per feature; this is a sensible
|
| 145 |
+
# default keyed on lowercase class labels.
|
| 146 |
+
_DEFAULT_FILL = "#A0A0A0"
|
| 147 |
+
_PALETTE = {
|
| 148 |
+
# ESRI 2020 LULC schema (terramind v1 base generative)
|
| 149 |
+
"water": "#1F77B4",
|
| 150 |
+
"trees": "#2CA02C",
|
| 151 |
+
"grass": "#7FBF53",
|
| 152 |
+
"flooded vegetation": "#74C476",
|
| 153 |
+
"crops": "#E1C75A",
|
| 154 |
+
"scrub/shrub": "#A6BC44",
|
| 155 |
+
"built": "#D62728",
|
| 156 |
+
"bare ground": "#B07A4C",
|
| 157 |
+
"snow/ice": "#E0E7EC",
|
| 158 |
+
"clouds": "#CCCCCC",
|
| 159 |
+
# NYC LoRA LULC schema
|
| 160 |
+
"cropland": "#E1C75A",
|
| 161 |
+
"bare": "#B07A4C",
|
| 162 |
+
# Buildings LoRA
|
| 163 |
+
"building": "#D62728",
|
| 164 |
+
"background": _DEFAULT_FILL,
|
| 165 |
+
}
|
app/context/dob_permits.py
ADDED
|
@@ -0,0 +1,258 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""NYC DOB construction-permit specialist β "what are they building".
|
| 2 |
+
|
| 3 |
+
Pulls active NYC DOB Permit Issuance records (Socrata `ipu4-2q9a`)
|
| 4 |
+
inside a polygon, filtered to recent New Building (NB), major
|
| 5 |
+
Alteration (A1), and Demolition (DM) jobs. Each project is then
|
| 6 |
+
cross-referenced against the static flood layers (Sandy 2012, DEP
|
| 7 |
+
Stormwater scenarios) so the reconciler can write things like:
|
| 8 |
+
|
| 9 |
+
"12 active major construction projects in Gowanus. Of these,
|
| 10 |
+
8 sit inside the DEP Extreme-2080 stormwater scenario."
|
| 11 |
+
|
| 12 |
+
The dataset uses separate gis_latitude / gis_longitude columns rather
|
| 13 |
+
than a Socrata Point, so we bbox-filter via SoQL then do exact
|
| 14 |
+
point-in-polygon containment client-side with shapely.
|
| 15 |
+
"""
|
| 16 |
+
from __future__ import annotations
|
| 17 |
+
|
| 18 |
+
import logging
|
| 19 |
+
from collections import Counter
|
| 20 |
+
from dataclasses import asdict, dataclass
|
| 21 |
+
from datetime import date, datetime, timedelta
|
| 22 |
+
from typing import Any
|
| 23 |
+
|
| 24 |
+
import geopandas as gpd
|
| 25 |
+
import httpx
|
| 26 |
+
from shapely.geometry import Point
|
| 27 |
+
|
| 28 |
+
log = logging.getLogger("riprap.dob_permits")
|
| 29 |
+
|
| 30 |
+
URL = "https://data.cityofnewyork.us/resource/ipu4-2q9a.json"
|
| 31 |
+
DOC_ID = "dob_permits"
|
| 32 |
+
CITATION = ("NYC DOB Permit Issuance (NYC OpenData ipu4-2q9a) β "
|
| 33 |
+
"issued/in-progress construction permits")
|
| 34 |
+
|
| 35 |
+
JOB_TYPE_LABELS = {
|
| 36 |
+
"NB": "new building",
|
| 37 |
+
"A1": "major alteration (use/occupancy)",
|
| 38 |
+
"A2": "minor alteration",
|
| 39 |
+
"A3": "minor work / interior",
|
| 40 |
+
"DM": "demolition",
|
| 41 |
+
"SG": "sign",
|
| 42 |
+
"PL": "plumbing",
|
| 43 |
+
"EQ": "equipment",
|
| 44 |
+
}
|
| 45 |
+
|
| 46 |
+
# Default filter: focus on "what are they building" β new construction,
|
| 47 |
+
# major alterations, demolitions. Skip minor mechanical permits.
|
| 48 |
+
DEFAULT_JOB_TYPES = ("NB", "A1", "DM")
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
@dataclass
|
| 52 |
+
class Permit:
|
| 53 |
+
job_id: str
|
| 54 |
+
job_type: str
|
| 55 |
+
job_type_label: str
|
| 56 |
+
permit_status: str
|
| 57 |
+
issuance_date: str
|
| 58 |
+
expiration_date: str | None
|
| 59 |
+
address: str
|
| 60 |
+
borough: str
|
| 61 |
+
bbl: str | None
|
| 62 |
+
lat: float
|
| 63 |
+
lon: float
|
| 64 |
+
owner_business: str | None
|
| 65 |
+
permittee_business: str | None
|
| 66 |
+
nta_name: str | None
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def permits_in_bbox(min_lat: float, min_lon: float,
|
| 70 |
+
max_lat: float, max_lon: float,
|
| 71 |
+
job_types: tuple[str, ...] = DEFAULT_JOB_TYPES,
|
| 72 |
+
since: date | None = None,
|
| 73 |
+
limit: int = 5000) -> list[Permit]:
|
| 74 |
+
"""Pull DOB permits intersecting a bounding box, recently issued, with
|
| 75 |
+
matching job types. We expand from polygon to bbox and rely on the
|
| 76 |
+
caller to do exact point-in-polygon filtering."""
|
| 77 |
+
if since is None:
|
| 78 |
+
since = date.today() - timedelta(days=540) # ~18 months
|
| 79 |
+
# gis_latitude/gis_longitude are stored as text in this dataset; cast
|
| 80 |
+
# to number for the bbox compare. issuance_date is a floating timestamp
|
| 81 |
+
# surfaced as 'MM/DD/YYYY' string β cast explicitly to floating_timestamp
|
| 82 |
+
# so the comparator parses ISO dates correctly. BETWEEN is picky on text
|
| 83 |
+
# columns, so use explicit >= / <= operators.
|
| 84 |
+
where = (
|
| 85 |
+
f"job_type IN ({','.join(repr(t) for t in job_types)})"
|
| 86 |
+
f" AND issuance_date::floating_timestamp >= '{since.isoformat()}'"
|
| 87 |
+
f" AND gis_latitude::number >= {min_lat}"
|
| 88 |
+
f" AND gis_latitude::number <= {max_lat}"
|
| 89 |
+
f" AND gis_longitude::number >= {min_lon}"
|
| 90 |
+
f" AND gis_longitude::number <= {max_lon}"
|
| 91 |
+
)
|
| 92 |
+
r = httpx.get(URL, params={
|
| 93 |
+
"$select": ",".join([
|
| 94 |
+
"job__", "job_type", "permit_status", "issuance_date",
|
| 95 |
+
"expiration_date", "house__", "street_name", "borough",
|
| 96 |
+
"block", "lot",
|
| 97 |
+
"gis_latitude", "gis_longitude", "owner_s_business_name",
|
| 98 |
+
"permittee_s_business_name", "gis_nta_name",
|
| 99 |
+
]),
|
| 100 |
+
"$where": where,
|
| 101 |
+
"$order": "issuance_date desc",
|
| 102 |
+
"$limit": str(limit),
|
| 103 |
+
}, timeout=60)
|
| 104 |
+
r.raise_for_status()
|
| 105 |
+
out: list[Permit] = []
|
| 106 |
+
for row in r.json():
|
| 107 |
+
try:
|
| 108 |
+
lat = float(row["gis_latitude"])
|
| 109 |
+
lon = float(row["gis_longitude"])
|
| 110 |
+
except (KeyError, ValueError, TypeError):
|
| 111 |
+
continue
|
| 112 |
+
addr = " ".join(filter(None, [
|
| 113 |
+
row.get("house__"),
|
| 114 |
+
(row.get("street_name") or "").title(),
|
| 115 |
+
])).strip()
|
| 116 |
+
# DOB has no `bbl` column; compose from borough + block + lot.
|
| 117 |
+
# Borough codes: MAN=1, BX=2, BK=3, QN=4, SI=5.
|
| 118 |
+
boro_code = {"MANHATTAN": "1", "BRONX": "2", "BROOKLYN": "3",
|
| 119 |
+
"QUEENS": "4", "STATEN ISLAND": "5"}.get(
|
| 120 |
+
(row.get("borough") or "").upper())
|
| 121 |
+
block = (row.get("block") or "").lstrip("0")
|
| 122 |
+
lot = (row.get("lot") or "").lstrip("0")
|
| 123 |
+
bbl = (f"{boro_code}-{block.zfill(5)}-{lot.zfill(4)}"
|
| 124 |
+
if boro_code and block and lot else None)
|
| 125 |
+
out.append(Permit(
|
| 126 |
+
job_id=row.get("job__", ""),
|
| 127 |
+
job_type=row.get("job_type", ""),
|
| 128 |
+
job_type_label=JOB_TYPE_LABELS.get(row.get("job_type", ""), row.get("job_type", "")),
|
| 129 |
+
permit_status=row.get("permit_status", ""),
|
| 130 |
+
issuance_date=(row.get("issuance_date") or "")[:10],
|
| 131 |
+
expiration_date=(row.get("expiration_date") or "")[:10] or None,
|
| 132 |
+
address=addr,
|
| 133 |
+
borough=(row.get("borough") or "").title(),
|
| 134 |
+
bbl=bbl,
|
| 135 |
+
lat=lat,
|
| 136 |
+
lon=lon,
|
| 137 |
+
owner_business=row.get("owner_s_business_name"),
|
| 138 |
+
permittee_business=row.get("permittee_s_business_name"),
|
| 139 |
+
nta_name=row.get("gis_nta_name"),
|
| 140 |
+
))
|
| 141 |
+
return out
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def permits_in_polygon(polygon, polygon_crs: str = "EPSG:4326",
|
| 145 |
+
job_types: tuple[str, ...] = DEFAULT_JOB_TYPES,
|
| 146 |
+
since: date | None = None) -> list[Permit]:
|
| 147 |
+
"""Permits inside a polygon. Uses bbox prefilter + shapely contains."""
|
| 148 |
+
g = gpd.GeoDataFrame(geometry=[polygon], crs=polygon_crs).to_crs("EPSG:4326")
|
| 149 |
+
geom = g.iloc[0].geometry
|
| 150 |
+
minx, miny, maxx, maxy = geom.bounds
|
| 151 |
+
raw = permits_in_bbox(miny, minx, maxy, maxx, job_types=job_types, since=since)
|
| 152 |
+
out: list[Permit] = []
|
| 153 |
+
for p in raw:
|
| 154 |
+
pt = Point(p.lon, p.lat)
|
| 155 |
+
if geom.contains(pt) or geom.intersects(pt):
|
| 156 |
+
out.append(p)
|
| 157 |
+
# Dedupe by job_id (one job can have multiple permits as work proceeds)
|
| 158 |
+
seen: dict[str, Permit] = {}
|
| 159 |
+
for p in out:
|
| 160 |
+
# Keep the most-recently-issued permit per job
|
| 161 |
+
cur = seen.get(p.job_id)
|
| 162 |
+
if cur is None or (p.issuance_date or "") > (cur.issuance_date or ""):
|
| 163 |
+
seen[p.job_id] = p
|
| 164 |
+
return list(seen.values())
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
def cross_reference_flood(permits: list[Permit]) -> list[dict[str, Any]]:
|
| 168 |
+
"""Tag each permit with which flood layers cover its point.
|
| 169 |
+
Adds: in_sandy (bool), dep_class (highest depth class hit across DEP scenarios),
|
| 170 |
+
dep_scenarios (list of scenario ids that fired)."""
|
| 171 |
+
if not permits:
|
| 172 |
+
return []
|
| 173 |
+
from app.flood_layers import dep_stormwater, sandy_inundation
|
| 174 |
+
pts = gpd.GeoDataFrame(
|
| 175 |
+
geometry=[Point(p.lon, p.lat) for p in permits],
|
| 176 |
+
crs="EPSG:4326",
|
| 177 |
+
).to_crs("EPSG:2263")
|
| 178 |
+
pts["_pid"] = list(range(len(pts)))
|
| 179 |
+
|
| 180 |
+
sandy_flags = sandy_inundation.join(pts).reset_index(drop=True).tolist()
|
| 181 |
+
|
| 182 |
+
dep_hits = {scen: dep_stormwater.join(pts, scen)["depth_class"].astype(int).tolist()
|
| 183 |
+
for scen in ("dep_extreme_2080", "dep_moderate_2050", "dep_moderate_current")}
|
| 184 |
+
|
| 185 |
+
out = []
|
| 186 |
+
for i, p in enumerate(permits):
|
| 187 |
+
scen_hits = {s: dep_hits[s][i] for s in dep_hits}
|
| 188 |
+
max_class = max(scen_hits.values(), default=0)
|
| 189 |
+
active_scens = [s for s, c in scen_hits.items() if c > 0]
|
| 190 |
+
out.append({
|
| 191 |
+
**asdict(p),
|
| 192 |
+
"in_sandy": bool(sandy_flags[i]),
|
| 193 |
+
"dep_max_class": max_class,
|
| 194 |
+
"dep_scenarios": active_scens,
|
| 195 |
+
"any_flood_layer_hit": bool(sandy_flags[i] or max_class > 0),
|
| 196 |
+
})
|
| 197 |
+
return out
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def summary_for_polygon(polygon, polygon_crs: str = "EPSG:4326",
|
| 201 |
+
since_days: int = 540,
|
| 202 |
+
top_n: int = 8) -> dict:
|
| 203 |
+
"""Full polygon-mode summary: list active permits, cross-reference each
|
| 204 |
+
with flood layers, return aggregate counts + a top-N projects-of-concern
|
| 205 |
+
list (those that hit at least one flood layer, ranked by max DEP class
|
| 206 |
+
+ Sandy hit)."""
|
| 207 |
+
since = date.today() - timedelta(days=since_days)
|
| 208 |
+
permits = permits_in_polygon(polygon, polygon_crs=polygon_crs, since=since)
|
| 209 |
+
enriched = cross_reference_flood(permits)
|
| 210 |
+
|
| 211 |
+
by_type: Counter = Counter(e["job_type_label"] for e in enriched)
|
| 212 |
+
by_status: Counter = Counter(e["permit_status"] for e in enriched)
|
| 213 |
+
n_total = len(enriched)
|
| 214 |
+
n_sandy = sum(1 for e in enriched if e["in_sandy"])
|
| 215 |
+
n_dep_any = sum(1 for e in enriched if e["dep_max_class"] > 0)
|
| 216 |
+
n_dep_severe = sum(1 for e in enriched if e["dep_max_class"] >= 2)
|
| 217 |
+
n_any_flood = sum(1 for e in enriched if e["any_flood_layer_hit"])
|
| 218 |
+
|
| 219 |
+
# Rank: severity = (in_sandy * 3) + dep_max_class
|
| 220 |
+
def severity(e):
|
| 221 |
+
return (3 if e["in_sandy"] else 0) + e["dep_max_class"]
|
| 222 |
+
flagged = sorted(
|
| 223 |
+
[e for e in enriched if e["any_flood_layer_hit"]],
|
| 224 |
+
key=severity, reverse=True,
|
| 225 |
+
)[:top_n]
|
| 226 |
+
|
| 227 |
+
# Light projection of every permit for map pinning (no need to ship the
|
| 228 |
+
# full permit record for the not-flagged ones β the map only needs lat,
|
| 229 |
+
# lon, address, job_type_label, and the flood-flag fields).
|
| 230 |
+
all_pins = [
|
| 231 |
+
{
|
| 232 |
+
"lat": e["lat"],
|
| 233 |
+
"lon": e["lon"],
|
| 234 |
+
"address": e["address"],
|
| 235 |
+
"job_type": e["job_type"],
|
| 236 |
+
"in_sandy": e["in_sandy"],
|
| 237 |
+
"dep_max_class": e["dep_max_class"],
|
| 238 |
+
"any_flood": e["any_flood_layer_hit"],
|
| 239 |
+
}
|
| 240 |
+
for e in enriched
|
| 241 |
+
]
|
| 242 |
+
return {
|
| 243 |
+
"since": since.isoformat(),
|
| 244 |
+
"n_total": n_total,
|
| 245 |
+
"n_in_sandy": n_sandy,
|
| 246 |
+
"n_in_dep_any": n_dep_any,
|
| 247 |
+
"n_in_dep_severe": n_dep_severe,
|
| 248 |
+
"n_any_flood": n_any_flood,
|
| 249 |
+
"by_job_type": dict(by_type.most_common()),
|
| 250 |
+
"by_permit_status":dict(by_status.most_common()),
|
| 251 |
+
"flagged_top": flagged,
|
| 252 |
+
"all_pins": all_pins,
|
| 253 |
+
"all_count": n_total,
|
| 254 |
+
}
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
def now_iso() -> str:
|
| 258 |
+
return datetime.utcnow().date().isoformat()
|
app/context/eo_chip_cache.py
ADDED
|
@@ -0,0 +1,345 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Per-query EO chip cache β Sentinel-2 L2A, Sentinel-1 RTC, DEM.
|
| 2 |
+
|
| 3 |
+
Fetches a co-registered (S2L2A, S1RTC, DEM) chip centered on (lat, lon)
|
| 4 |
+
and returns a dict of torch tensors ready for TerraMind-NYC inference.
|
| 5 |
+
The TerraMind base was trained with `temporal_n_timestamps=4`, so this
|
| 6 |
+
helper expands a single S2/S1 acquisition to T=4 by repetition along
|
| 7 |
+
the temporal axis. Single-timestep nowcasting trades some training-
|
| 8 |
+
distribution match for a much simpler runtime β the published LoRA
|
| 9 |
+
adapters still produce sensible argmax masks at T=1 / tiled.
|
| 10 |
+
|
| 11 |
+
Failure semantics mirror prithvi_live: every dependency or network
|
| 12 |
+
failure is converted to a clean `{ok: False, skipped: <reason>}`
|
| 13 |
+
result, never a raised exception. Callers (FSM specialists) that
|
| 14 |
+
chain off the chip can short-circuit on `ok=False` and skip the
|
| 15 |
+
specialist instead of surfacing a noisy error.
|
| 16 |
+
"""
|
| 17 |
+
from __future__ import annotations
|
| 18 |
+
|
| 19 |
+
import concurrent.futures
|
| 20 |
+
import logging
|
| 21 |
+
import os
|
| 22 |
+
import threading
|
| 23 |
+
import time
|
| 24 |
+
from typing import Any
|
| 25 |
+
|
| 26 |
+
log = logging.getLogger("riprap.eo_chip_cache")
|
| 27 |
+
|
| 28 |
+
ENABLE = os.environ.get("RIPRAP_EO_CHIP_ENABLE", "1").lower() in ("1", "true", "yes")
|
| 29 |
+
SEARCH_DAYS = int(os.environ.get("RIPRAP_EO_CHIP_SEARCH_DAYS", "120"))
|
| 30 |
+
MAX_CLOUD_PCT = float(os.environ.get("RIPRAP_EO_CHIP_MAX_CLOUD", "30"))
|
| 31 |
+
CHIP_PX = int(os.environ.get("RIPRAP_EO_CHIP_PX", "224"))
|
| 32 |
+
PIXEL_M = 10
|
| 33 |
+
N_TIMESTEPS = 4
|
| 34 |
+
|
| 35 |
+
# 12-band S2 L2A in TerraMind's expected order.
|
| 36 |
+
S2_BANDS = ["B01", "B02", "B03", "B04", "B05", "B06", "B07",
|
| 37 |
+
"B08", "B8A", "B09", "B11", "B12"]
|
| 38 |
+
|
| 39 |
+
# Sentinel-1 RTC on Planetary Computer publishes vv/vh polarisations.
|
| 40 |
+
S1_BANDS = ["vv", "vh"]
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def _has_required_deps() -> tuple[bool, str | None]:
|
| 44 |
+
missing: list[str] = []
|
| 45 |
+
for name in ("planetary_computer", "pystac_client",
|
| 46 |
+
"rioxarray", "xarray", "torch", "numpy"):
|
| 47 |
+
try:
|
| 48 |
+
__import__(name)
|
| 49 |
+
except ImportError:
|
| 50 |
+
missing.append(name)
|
| 51 |
+
if missing:
|
| 52 |
+
return False, ", ".join(missing)
|
| 53 |
+
return True, None
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
_DEPS_OK, _DEPS_MISSING = _has_required_deps()
|
| 57 |
+
_FETCH_LOCK = threading.Lock()
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def _search_s2(lat: float, lon: float):
|
| 61 |
+
"""Return (item, cloud_cover) for the most recent low-cloud S2L2A
|
| 62 |
+
acquisition near (lat, lon), or (None, None) if no scene exists."""
|
| 63 |
+
import datetime as dt
|
| 64 |
+
|
| 65 |
+
import planetary_computer as pc
|
| 66 |
+
from pystac_client import Client
|
| 67 |
+
end = dt.datetime.utcnow().date()
|
| 68 |
+
start = end - dt.timedelta(days=SEARCH_DAYS)
|
| 69 |
+
client = Client.open(
|
| 70 |
+
"https://planetarycomputer.microsoft.com/api/stac/v1",
|
| 71 |
+
modifier=pc.sign_inplace,
|
| 72 |
+
)
|
| 73 |
+
delta = 0.02
|
| 74 |
+
search = client.search(
|
| 75 |
+
collections=["sentinel-2-l2a"],
|
| 76 |
+
bbox=[lon - delta, lat - delta, lon + delta, lat + delta],
|
| 77 |
+
datetime=f"{start}/{end}",
|
| 78 |
+
query={"eo:cloud_cover": {"lt": MAX_CLOUD_PCT}},
|
| 79 |
+
max_items=20,
|
| 80 |
+
)
|
| 81 |
+
items = sorted(
|
| 82 |
+
search.items(),
|
| 83 |
+
key=lambda it: (it.properties.get("eo:cloud_cover", 100),
|
| 84 |
+
-(it.datetime.timestamp() if it.datetime else 0)),
|
| 85 |
+
)
|
| 86 |
+
if not items:
|
| 87 |
+
return None, None
|
| 88 |
+
item = items[0]
|
| 89 |
+
cc = float(item.properties.get("eo:cloud_cover", -1))
|
| 90 |
+
return item, cc
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def _search_s1(item_dt, lat: float, lon: float):
|
| 94 |
+
"""Return the closest Sentinel-1 RTC acquisition to the given S2
|
| 95 |
+
datetime, or None if Planetary Computer has nothing nearby."""
|
| 96 |
+
import datetime as dt
|
| 97 |
+
|
| 98 |
+
import planetary_computer as pc
|
| 99 |
+
from pystac_client import Client
|
| 100 |
+
win = dt.timedelta(days=10)
|
| 101 |
+
start = item_dt - win
|
| 102 |
+
end = item_dt + win
|
| 103 |
+
client = Client.open(
|
| 104 |
+
"https://planetarycomputer.microsoft.com/api/stac/v1",
|
| 105 |
+
modifier=pc.sign_inplace,
|
| 106 |
+
)
|
| 107 |
+
delta = 0.02
|
| 108 |
+
search = client.search(
|
| 109 |
+
collections=["sentinel-1-rtc"],
|
| 110 |
+
bbox=[lon - delta, lat - delta, lon + delta, lat + delta],
|
| 111 |
+
datetime=f"{start.isoformat()}/{end.isoformat()}",
|
| 112 |
+
max_items=10,
|
| 113 |
+
)
|
| 114 |
+
items = list(search.items())
|
| 115 |
+
if not items:
|
| 116 |
+
return None
|
| 117 |
+
items.sort(key=lambda it:
|
| 118 |
+
abs((it.datetime - item_dt).total_seconds())
|
| 119 |
+
if it.datetime else 1e18)
|
| 120 |
+
return items[0]
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def _read_band(href, bbox_xy_meters, epsg):
|
| 124 |
+
"""Read a single COG band, clipped to the bbox, and resample to
|
| 125 |
+
CHIP_PX Γ CHIP_PX. Returns a numpy array (CHIP_PX, CHIP_PX) float32.
|
| 126 |
+
"""
|
| 127 |
+
import numpy as np
|
| 128 |
+
import rioxarray # noqa: F401
|
| 129 |
+
da = rioxarray.open_rasterio(href, masked=False).squeeze(drop=True)
|
| 130 |
+
da = da.rio.clip_box(minx=bbox_xy_meters[0], miny=bbox_xy_meters[1],
|
| 131 |
+
maxx=bbox_xy_meters[2], maxy=bbox_xy_meters[3])
|
| 132 |
+
if da.shape[-2] != CHIP_PX or da.shape[-1] != CHIP_PX:
|
| 133 |
+
# Resample (nearest is fine for the 10/20/60 m S2 mix; S1 is 10 m,
|
| 134 |
+
# DEM is 30 m and benefits from bilinear; we keep nearest for
|
| 135 |
+
# simplicity β the TerraMind LoRA was trained against terratorch's
|
| 136 |
+
# default resampler which is also nearest).
|
| 137 |
+
da = da.rio.reproject(
|
| 138 |
+
f"EPSG:{epsg}", shape=(CHIP_PX, CHIP_PX), resampling=0
|
| 139 |
+
)
|
| 140 |
+
arr = da.values.astype("float32")
|
| 141 |
+
return np.nan_to_num(arr)
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def _fetch_modalities(lat: float, lon: float, timeout_s: float = 60.0) -> dict[str, Any]:
|
| 145 |
+
"""Fetch S2L2A + S1RTC + DEM as numpy arrays, resampled to a common
|
| 146 |
+
CHIP_PX Γ CHIP_PX grid centered on (lat, lon).
|
| 147 |
+
"""
|
| 148 |
+
import numpy as np
|
| 149 |
+
from pyproj import Transformer
|
| 150 |
+
|
| 151 |
+
t0 = time.time()
|
| 152 |
+
item, cc = _search_s2(lat, lon)
|
| 153 |
+
if item is None:
|
| 154 |
+
return {"ok": False,
|
| 155 |
+
"skipped": f"no <{MAX_CLOUD_PCT}% cloud S2 in last "
|
| 156 |
+
f"{SEARCH_DAYS}d"}
|
| 157 |
+
if "proj:epsg" in item.properties:
|
| 158 |
+
epsg = int(item.properties["proj:epsg"])
|
| 159 |
+
else:
|
| 160 |
+
code = item.properties.get("proj:code", "")
|
| 161 |
+
if not code.startswith("EPSG:"):
|
| 162 |
+
return {"ok": False,
|
| 163 |
+
"skipped": "STAC item missing proj:epsg / proj:code"}
|
| 164 |
+
epsg = int(code.split(":", 1)[1])
|
| 165 |
+
|
| 166 |
+
fwd = Transformer.from_crs("EPSG:4326", f"EPSG:{epsg}", always_xy=True)
|
| 167 |
+
cx, cy = fwd.transform(lon, lat)
|
| 168 |
+
half_m = CHIP_PX / 2 * PIXEL_M
|
| 169 |
+
bbox = (cx - half_m, cy - half_m, cx + half_m, cy + half_m)
|
| 170 |
+
|
| 171 |
+
if time.time() - t0 > timeout_s:
|
| 172 |
+
return {"ok": False, "skipped": "STAC search exceeded budget"}
|
| 173 |
+
|
| 174 |
+
# ---- S2L2A: 12 bands ------------------------------------------------
|
| 175 |
+
s2_arrs = []
|
| 176 |
+
try:
|
| 177 |
+
for b in S2_BANDS:
|
| 178 |
+
href = item.assets[b].href
|
| 179 |
+
s2_arrs.append(_read_band(href, bbox, epsg))
|
| 180 |
+
except Exception as e:
|
| 181 |
+
log.warning("eo_chip: S2 band fetch failed (%s); aborting", e)
|
| 182 |
+
return {"ok": False, "err": f"S2 fetch failed: {type(e).__name__}: {e}"}
|
| 183 |
+
s2 = np.stack(s2_arrs) # (12, H, W)
|
| 184 |
+
if s2.mean() > 1.0:
|
| 185 |
+
s2 = s2 / 10000.0 # scale L2A reflectance from int16 to ~[0, 1]
|
| 186 |
+
|
| 187 |
+
# ---- S1RTC: 2 polarisations (best effort) ---------------------------
|
| 188 |
+
s1: np.ndarray | None = None
|
| 189 |
+
s1_meta: dict[str, Any] = {}
|
| 190 |
+
if time.time() - t0 < timeout_s:
|
| 191 |
+
try:
|
| 192 |
+
s1_item = _search_s1(item.datetime, lat, lon)
|
| 193 |
+
if s1_item is not None:
|
| 194 |
+
s1_arrs = []
|
| 195 |
+
for b in S1_BANDS:
|
| 196 |
+
href = s1_item.assets[b].href
|
| 197 |
+
s1_arrs.append(_read_band(href, bbox, epsg))
|
| 198 |
+
s1 = np.stack(s1_arrs)
|
| 199 |
+
s1_meta = {
|
| 200 |
+
"scene_id": s1_item.id,
|
| 201 |
+
"datetime": (s1_item.datetime.isoformat()
|
| 202 |
+
if s1_item.datetime else None),
|
| 203 |
+
}
|
| 204 |
+
except Exception as e:
|
| 205 |
+
log.warning("eo_chip: S1 fetch best-effort failed: %s", e)
|
| 206 |
+
|
| 207 |
+
# ---- DEM: Copernicus 30 m via planetary_computer (best effort) ------
|
| 208 |
+
dem: np.ndarray | None = None
|
| 209 |
+
if time.time() - t0 < timeout_s:
|
| 210 |
+
try:
|
| 211 |
+
import planetary_computer as pc
|
| 212 |
+
from pystac_client import Client
|
| 213 |
+
client = Client.open(
|
| 214 |
+
"https://planetarycomputer.microsoft.com/api/stac/v1",
|
| 215 |
+
modifier=pc.sign_inplace,
|
| 216 |
+
)
|
| 217 |
+
dem_search = client.search(
|
| 218 |
+
collections=["cop-dem-glo-30"],
|
| 219 |
+
bbox=[lon - 0.02, lat - 0.02, lon + 0.02, lat + 0.02],
|
| 220 |
+
max_items=1,
|
| 221 |
+
)
|
| 222 |
+
dem_items = list(dem_search.items())
|
| 223 |
+
if dem_items:
|
| 224 |
+
href = dem_items[0].assets["data"].href
|
| 225 |
+
dem = _read_band(href, bbox, epsg)
|
| 226 |
+
dem = dem[None, :, :] # add channel dim
|
| 227 |
+
except Exception as e:
|
| 228 |
+
log.warning("eo_chip: DEM fetch best-effort failed: %s", e)
|
| 229 |
+
|
| 230 |
+
return {
|
| 231 |
+
"ok": True,
|
| 232 |
+
"lat": lat, "lon": lon,
|
| 233 |
+
"epsg": epsg, "chip_px": CHIP_PX, "pixel_m": PIXEL_M,
|
| 234 |
+
"s2": s2, "s1": s1, "dem": dem,
|
| 235 |
+
"s2_meta": {
|
| 236 |
+
"scene_id": item.id,
|
| 237 |
+
"datetime": (item.datetime.isoformat() if item.datetime else None),
|
| 238 |
+
"cloud_cover": cc,
|
| 239 |
+
},
|
| 240 |
+
"s1_meta": s1_meta,
|
| 241 |
+
"elapsed_s": round(time.time() - t0, 2),
|
| 242 |
+
}
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
def _to_terramind_tensors(modalities: dict[str, Any]) -> dict[str, Any]:
|
| 246 |
+
"""Shape numpy modality arrays into the (B, C, T, H, W) tensors
|
| 247 |
+
TerraMind expects with `temporal_n_timestamps=4`. Single-timestep
|
| 248 |
+
fetches get tiled to T=4 β same observation in every slot.
|
| 249 |
+
"""
|
| 250 |
+
import torch
|
| 251 |
+
s2 = modalities["s2"] # (12, H, W)
|
| 252 |
+
s2_t = torch.from_numpy(s2).float().unsqueeze(1) # (12, 1, H, W)
|
| 253 |
+
s2_t = s2_t.repeat(1, N_TIMESTEPS, 1, 1).unsqueeze(0) # (1, 12, T, H, W)
|
| 254 |
+
chips = {"S2L2A": s2_t}
|
| 255 |
+
if modalities.get("s1") is not None:
|
| 256 |
+
s1 = modalities["s1"] # (2, H, W)
|
| 257 |
+
s1_t = torch.from_numpy(s1).float().unsqueeze(1)
|
| 258 |
+
s1_t = s1_t.repeat(1, N_TIMESTEPS, 1, 1).unsqueeze(0)
|
| 259 |
+
chips["S1RTC"] = s1_t
|
| 260 |
+
if modalities.get("dem") is not None:
|
| 261 |
+
dem = modalities["dem"] # (1, H, W)
|
| 262 |
+
dem_t = torch.from_numpy(dem).float().unsqueeze(1)
|
| 263 |
+
dem_t = dem_t.repeat(1, N_TIMESTEPS, 1, 1).unsqueeze(0)
|
| 264 |
+
chips["DEM"] = dem_t
|
| 265 |
+
return chips
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
def _fetch_and_build(lat: float, lon: float, timeout_s: float) -> dict[str, Any]:
|
| 269 |
+
"""Inner fetch + tensor build, run inside a bounded thread."""
|
| 270 |
+
with _FETCH_LOCK:
|
| 271 |
+
try:
|
| 272 |
+
modalities = _fetch_modalities(lat, lon, timeout_s=timeout_s)
|
| 273 |
+
except Exception as e:
|
| 274 |
+
log.exception("eo_chip: fetch failed")
|
| 275 |
+
return {"ok": False, "err": f"{type(e).__name__}: {e}"}
|
| 276 |
+
if not modalities.get("ok"):
|
| 277 |
+
return modalities
|
| 278 |
+
try:
|
| 279 |
+
modalities["tensors"] = _to_terramind_tensors(modalities)
|
| 280 |
+
except Exception as e:
|
| 281 |
+
log.exception("eo_chip: tensor build failed")
|
| 282 |
+
return {"ok": False,
|
| 283 |
+
"err": f"tensor build failed: {type(e).__name__}: {e}"}
|
| 284 |
+
# Compute the chip's WGS84 bbox so downstream TerraMind specialists
|
| 285 |
+
# can polygonise their predictions onto the map. The chip is
|
| 286 |
+
# CHIP_PX Γ CHIP_PX at PIXEL_M (10 m) in the scene's UTM zone;
|
| 287 |
+
# reproject the four corners to EPSG:4326 and use the
|
| 288 |
+
# axis-aligned envelope.
|
| 289 |
+
try:
|
| 290 |
+
from pyproj import Transformer
|
| 291 |
+
half_m = (CHIP_PX * PIXEL_M) / 2.0
|
| 292 |
+
t_to_utm = Transformer.from_crs(
|
| 293 |
+
"EPSG:4326", f"EPSG:{modalities['epsg']}", always_xy=True)
|
| 294 |
+
t_to_4326 = Transformer.from_crs(
|
| 295 |
+
f"EPSG:{modalities['epsg']}", "EPSG:4326", always_xy=True)
|
| 296 |
+
cx, cy = t_to_utm.transform(lon, lat)
|
| 297 |
+
corners_utm = [
|
| 298 |
+
(cx - half_m, cy - half_m),
|
| 299 |
+
(cx - half_m, cy + half_m),
|
| 300 |
+
(cx + half_m, cy - half_m),
|
| 301 |
+
(cx + half_m, cy + half_m),
|
| 302 |
+
]
|
| 303 |
+
corners_ll = [t_to_4326.transform(x, y) for x, y in corners_utm]
|
| 304 |
+
lons = [c[0] for c in corners_ll]
|
| 305 |
+
lats = [c[1] for c in corners_ll]
|
| 306 |
+
modalities["bounds_4326"] = (
|
| 307 |
+
min(lons), min(lats), max(lons), max(lats))
|
| 308 |
+
except Exception:
|
| 309 |
+
log.exception("eo_chip: bounds_4326 reprojection failed")
|
| 310 |
+
return modalities
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
def fetch(lat: float, lon: float, timeout_s: float = 60.0) -> dict[str, Any]:
|
| 314 |
+
"""Run the chip pipeline. Always returns a dict with at minimum
|
| 315 |
+
`{ok, skipped|err, ...}`; on success the dict carries the
|
| 316 |
+
co-registered numpy arrays plus `tensors` (the TerraMind-shaped
|
| 317 |
+
torch dict).
|
| 318 |
+
|
| 319 |
+
Runs in a daemon thread so that STAC searches and COG band downloads
|
| 320 |
+
(which use requests/rioxarray without per-call timeouts) are bounded
|
| 321 |
+
by a hard wall-clock deadline even when the network hangs.
|
| 322 |
+
"""
|
| 323 |
+
if not ENABLE:
|
| 324 |
+
return {"ok": False, "skipped": "RIPRAP_EO_CHIP_ENABLE=0"}
|
| 325 |
+
if not _DEPS_OK:
|
| 326 |
+
return {"ok": False,
|
| 327 |
+
"skipped": f"deps unavailable on this deployment: "
|
| 328 |
+
f"{_DEPS_MISSING}"}
|
| 329 |
+
# Hard wall-clock cap: pystac_client / rioxarray COG reads don't expose
|
| 330 |
+
# uniform per-request timeouts, so we bound the whole pipeline here.
|
| 331 |
+
hard_timeout = timeout_s + 15.0
|
| 332 |
+
# Propagate the parent thread's emissions tracker into the worker so
|
| 333 |
+
# any inference._post calls made inside _fetch_and_build are recorded.
|
| 334 |
+
from app import emissions as _emissions
|
| 335 |
+
_parent_tracker = _emissions.current()
|
| 336 |
+
with concurrent.futures.ThreadPoolExecutor(
|
| 337 |
+
max_workers=1,
|
| 338 |
+
initializer=lambda t=_parent_tracker: _emissions.install(t),
|
| 339 |
+
) as pool:
|
| 340 |
+
future = pool.submit(_fetch_and_build, lat, lon, timeout_s)
|
| 341 |
+
try:
|
| 342 |
+
return future.result(timeout=hard_timeout)
|
| 343 |
+
except concurrent.futures.TimeoutError:
|
| 344 |
+
log.warning("eo_chip: hard timeout after %.0fs (STAC/COG hung)", hard_timeout)
|
| 345 |
+
return {"ok": False, "skipped": f"eo_chip timed out after {hard_timeout:.0f}s"}
|
app/context/floodnet.py
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FloodNet NYC β live ultrasonic flood sensor network.
|
| 2 |
+
|
| 3 |
+
Hasura GraphQL endpoint, no auth, ~350 sensors. Used for:
|
| 4 |
+
- sensors_near(lat, lon, radius_m) β list of deployments
|
| 5 |
+
- flood_events_for(deployment_ids, since) β labeled flood events per sensor
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from dataclasses import dataclass
|
| 10 |
+
from datetime import datetime, timedelta, timezone
|
| 11 |
+
from typing import Any
|
| 12 |
+
|
| 13 |
+
import httpx
|
| 14 |
+
|
| 15 |
+
URL = "https://api.floodnet.nyc/v1/graphql"
|
| 16 |
+
DOC_ID = "floodnet"
|
| 17 |
+
CITATION = "FloodNet NYC ultrasonic depth sensors (api.floodnet.nyc)"
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
@dataclass
|
| 21 |
+
class Sensor:
|
| 22 |
+
deployment_id: str
|
| 23 |
+
name: str
|
| 24 |
+
street: str
|
| 25 |
+
borough: str
|
| 26 |
+
status: str
|
| 27 |
+
deployed_at: str | None
|
| 28 |
+
lat: float | None = None
|
| 29 |
+
lon: float | None = None
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
@dataclass
|
| 33 |
+
class FloodEvent:
|
| 34 |
+
deployment_id: str
|
| 35 |
+
start_time: str
|
| 36 |
+
end_time: str | None
|
| 37 |
+
max_depth_mm: int | None
|
| 38 |
+
label: str | None
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _gql(query: str, variables: dict[str, Any]) -> dict:
|
| 42 |
+
r = httpx.post(URL, json={"query": query, "variables": variables},
|
| 43 |
+
timeout=20, verify=False)
|
| 44 |
+
r.raise_for_status()
|
| 45 |
+
j = r.json()
|
| 46 |
+
if "errors" in j:
|
| 47 |
+
raise RuntimeError(f"FloodNet GraphQL error: {j['errors']}")
|
| 48 |
+
return j["data"]
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
_NEAR_Q = """
|
| 52 |
+
query Near($lat: Float!, $lon: Float!, $r: Float!) {
|
| 53 |
+
deployments_within_radius(args:{lat:$lat, lon:$lon, radius_meters:$r},
|
| 54 |
+
order_by:{date_deployed: asc}) {
|
| 55 |
+
deployment_id
|
| 56 |
+
name
|
| 57 |
+
sensor_address_street
|
| 58 |
+
sensor_address_borough
|
| 59 |
+
sensor_status
|
| 60 |
+
date_deployed
|
| 61 |
+
location
|
| 62 |
+
}
|
| 63 |
+
}"""
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _parse_location(loc) -> tuple[float | None, float | None]:
|
| 67 |
+
"""Hasura PostGIS geometry returned as a GeoJSON object."""
|
| 68 |
+
if not loc or not isinstance(loc, dict):
|
| 69 |
+
return None, None
|
| 70 |
+
coords = loc.get("coordinates")
|
| 71 |
+
if not coords or len(coords) < 2:
|
| 72 |
+
return None, None
|
| 73 |
+
return coords[1], coords[0] # (lat, lon) from (lon, lat)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def sensors_near(lat: float, lon: float, radius_m: float = 1000) -> list[Sensor]:
|
| 77 |
+
d = _gql(_NEAR_Q, {"lat": lat, "lon": lon, "r": radius_m})
|
| 78 |
+
out = []
|
| 79 |
+
for row in d["deployments_within_radius"]:
|
| 80 |
+
slat, slon = _parse_location(row.get("location"))
|
| 81 |
+
out.append(Sensor(
|
| 82 |
+
deployment_id=row["deployment_id"],
|
| 83 |
+
name=row["name"] or "",
|
| 84 |
+
street=row.get("sensor_address_street") or "",
|
| 85 |
+
borough=row.get("sensor_address_borough") or "",
|
| 86 |
+
status=row.get("sensor_status") or "",
|
| 87 |
+
deployed_at=row.get("date_deployed"),
|
| 88 |
+
lat=slat,
|
| 89 |
+
lon=slon,
|
| 90 |
+
))
|
| 91 |
+
return out
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
_EVENTS_Q = """
|
| 95 |
+
query Events($ids: [String!], $since: timestamp!) {
|
| 96 |
+
sensor_events(where:{
|
| 97 |
+
deployment_id:{_in:$ids},
|
| 98 |
+
start_time:{_gte:$since},
|
| 99 |
+
label:{_eq:"flood"}
|
| 100 |
+
}, order_by:{start_time: desc}, limit: 200) {
|
| 101 |
+
deployment_id
|
| 102 |
+
start_time
|
| 103 |
+
end_time
|
| 104 |
+
max_depth_proc_mm
|
| 105 |
+
label
|
| 106 |
+
}
|
| 107 |
+
}"""
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def flood_events_for(deployment_ids: list[str],
|
| 111 |
+
since: datetime | None = None) -> list[FloodEvent]:
|
| 112 |
+
if not deployment_ids:
|
| 113 |
+
return []
|
| 114 |
+
if since is None:
|
| 115 |
+
since = datetime.now(timezone.utc) - timedelta(days=365 * 3)
|
| 116 |
+
d = _gql(_EVENTS_Q, {
|
| 117 |
+
"ids": deployment_ids,
|
| 118 |
+
"since": since.isoformat(timespec="seconds").replace("+00:00", ""),
|
| 119 |
+
})
|
| 120 |
+
return [
|
| 121 |
+
FloodEvent(
|
| 122 |
+
deployment_id=row["deployment_id"],
|
| 123 |
+
start_time=row["start_time"],
|
| 124 |
+
end_time=row.get("end_time"),
|
| 125 |
+
max_depth_mm=row.get("max_depth_proc_mm"),
|
| 126 |
+
label=row.get("label"),
|
| 127 |
+
)
|
| 128 |
+
for row in d["sensor_events"]
|
| 129 |
+
]
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def summary_for_point(lat: float, lon: float, radius_m: float = 600) -> dict:
|
| 133 |
+
"""One-shot summary used by the FSM node and the cited paragraph."""
|
| 134 |
+
sensors = sensors_near(lat, lon, radius_m)
|
| 135 |
+
ids = [s.deployment_id for s in sensors]
|
| 136 |
+
events = flood_events_for(ids)
|
| 137 |
+
by_dep: dict[str, list[FloodEvent]] = {}
|
| 138 |
+
for e in events:
|
| 139 |
+
by_dep.setdefault(e.deployment_id, []).append(e)
|
| 140 |
+
peak = max((e for e in events if e.max_depth_mm is not None),
|
| 141 |
+
key=lambda e: e.max_depth_mm or 0, default=None)
|
| 142 |
+
return {
|
| 143 |
+
"n_sensors": len(sensors),
|
| 144 |
+
"sensors": [vars(s) for s in sensors],
|
| 145 |
+
"n_flood_events_3y": len(events),
|
| 146 |
+
"n_sensors_with_events": len(by_dep),
|
| 147 |
+
"peak_event": vars(peak) if peak else None,
|
| 148 |
+
}
|
app/context/gliner_extract.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""GLiNER (urchade/gliner_medium-v2.1) typed-entity extraction over the
|
| 2 |
+
RAG retriever's top paragraphs.
|
| 3 |
+
|
| 4 |
+
Adds structured fields to the reconciler's grounding context. For each
|
| 5 |
+
RAG chunk the specialist emits, GLiNER produces a list of typed spans
|
| 6 |
+
with one of five labels:
|
| 7 |
+
|
| 8 |
+
nyc_location (e.g. "Coney Island")
|
| 9 |
+
dollar_amount (e.g. "$5.6 million")
|
| 10 |
+
date_range (e.g. "fiscal year 2025-2027")
|
| 11 |
+
agency (e.g. "NYC DEP")
|
| 12 |
+
infrastructure_project (e.g. "Bluebelt expansion")
|
| 13 |
+
|
| 14 |
+
The doc_id for emission is `gliner_<source>` where `<source>` is the
|
| 15 |
+
RAG chunk's doc_id stripped of its `rag_` prefix. So `rag_comptroller`
|
| 16 |
+
becomes `gliner_comptroller`. The reconciler can then cite typed
|
| 17 |
+
fields with `[gliner_comptroller]`.
|
| 18 |
+
|
| 19 |
+
License: Apache-2.0 β `urchade/gliner_medium-v2.1` (NOT the
|
| 20 |
+
`gliner_base` variant, which is CC-BY-NC-4.0). See
|
| 21 |
+
experiments/shared/licenses.md.
|
| 22 |
+
"""
|
| 23 |
+
|
| 24 |
+
from __future__ import annotations
|
| 25 |
+
|
| 26 |
+
import logging
|
| 27 |
+
import os
|
| 28 |
+
from dataclasses import dataclass
|
| 29 |
+
|
| 30 |
+
log = logging.getLogger("riprap.gliner")
|
| 31 |
+
|
| 32 |
+
ENTITY_LABELS = [
|
| 33 |
+
"nyc_location",
|
| 34 |
+
"dollar_amount",
|
| 35 |
+
"date_range",
|
| 36 |
+
"agency",
|
| 37 |
+
"infrastructure_project",
|
| 38 |
+
]
|
| 39 |
+
|
| 40 |
+
DEFAULT_THRESHOLD = float(os.environ.get("RIPRAP_GLINER_THRESHOLD", "0.45"))
|
| 41 |
+
MODEL_NAME = os.environ.get("RIPRAP_GLINER_MODEL", "urchade/gliner_medium-v2.1")
|
| 42 |
+
ENABLE = os.environ.get("RIPRAP_GLINER_ENABLE", "1").lower() in ("1", "true", "yes")
|
| 43 |
+
|
| 44 |
+
_MODEL = None # lazy
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
@dataclass
|
| 48 |
+
class Extraction:
|
| 49 |
+
label: str
|
| 50 |
+
text: str
|
| 51 |
+
score: float
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _ensure_model():
|
| 55 |
+
"""Lazy GLiNER load. Returns None if disabled or load fails so
|
| 56 |
+
callers can silently fall back to no-op."""
|
| 57 |
+
global _MODEL
|
| 58 |
+
if not ENABLE:
|
| 59 |
+
return None
|
| 60 |
+
if _MODEL is not None:
|
| 61 |
+
return _MODEL
|
| 62 |
+
try:
|
| 63 |
+
from gliner import GLiNER
|
| 64 |
+
log.info("gliner: loading %s", MODEL_NAME)
|
| 65 |
+
_MODEL = GLiNER.from_pretrained(MODEL_NAME)
|
| 66 |
+
except Exception:
|
| 67 |
+
log.exception("gliner: load failed; specialist will no-op")
|
| 68 |
+
_MODEL = False # sentinel
|
| 69 |
+
return _MODEL or None
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def warm():
|
| 73 |
+
_ensure_model()
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def _source_short(rag_doc_id: str) -> str:
|
| 77 |
+
"""`rag_comptroller` -> `comptroller`. Anything not prefixed `rag_`
|
| 78 |
+
passes through unchanged."""
|
| 79 |
+
return rag_doc_id[4:] if rag_doc_id.startswith("rag_") else rag_doc_id
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def extract_for_chunk(text: str, threshold: float = DEFAULT_THRESHOLD) -> list[Extraction]:
|
| 83 |
+
if not text:
|
| 84 |
+
return []
|
| 85 |
+
|
| 86 |
+
# v0.4.5 β try the MI300X service first. The remote handles its
|
| 87 |
+
# own GLiNER load; this lets cpu-basic surfaces run typed
|
| 88 |
+
# extraction without baking gliner into the image.
|
| 89 |
+
try:
|
| 90 |
+
from app import inference as _inf
|
| 91 |
+
if _inf.remote_enabled():
|
| 92 |
+
remote = _inf.gliner_extract(text, ENTITY_LABELS)
|
| 93 |
+
if remote.get("ok"):
|
| 94 |
+
return [
|
| 95 |
+
Extraction(label=e["label"], text=e["text"],
|
| 96 |
+
score=float(e.get("score", 0)))
|
| 97 |
+
for e in remote.get("entities", [])
|
| 98 |
+
if e.get("score", 0) >= threshold
|
| 99 |
+
]
|
| 100 |
+
except _inf.RemoteUnreachable as e:
|
| 101 |
+
log.info("gliner: remote unreachable (%s); local fallback", e)
|
| 102 |
+
except Exception:
|
| 103 |
+
log.exception("gliner: remote call failed; local fallback")
|
| 104 |
+
|
| 105 |
+
model = _ensure_model()
|
| 106 |
+
if model is None:
|
| 107 |
+
return []
|
| 108 |
+
raw = model.predict_entities(text, ENTITY_LABELS, threshold=threshold)
|
| 109 |
+
return [Extraction(label=r["label"], text=r["text"],
|
| 110 |
+
score=float(r["score"])) for r in raw]
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def extract_for_rag_hits(hits: list[dict],
|
| 114 |
+
threshold: float = DEFAULT_THRESHOLD,
|
| 115 |
+
max_hits: int = 3) -> dict[str, dict]:
|
| 116 |
+
"""Run GLiNER on the top-`max_hits` RAG hits. Returns a dict keyed by
|
| 117 |
+
short source id (e.g. "comptroller") with the structured payload
|
| 118 |
+
that the FSM stores into state["gliner"] and that
|
| 119 |
+
reconcile.build_documents() consumes."""
|
| 120 |
+
out: dict[str, dict] = {}
|
| 121 |
+
if not hits:
|
| 122 |
+
return out
|
| 123 |
+
for h in hits[:max_hits]:
|
| 124 |
+
source = _source_short(h.get("doc_id", "rag_unknown"))
|
| 125 |
+
ents = extract_for_chunk(h.get("text", ""), threshold=threshold)
|
| 126 |
+
if not ents:
|
| 127 |
+
continue
|
| 128 |
+
# Dedup verbatim repeats (common in agency PDFs that repeat
|
| 129 |
+
# "DEP" 13 times in a methodology section).
|
| 130 |
+
seen = set()
|
| 131 |
+
deduped: list[Extraction] = []
|
| 132 |
+
for e in ents:
|
| 133 |
+
key = (e.label, e.text.lower())
|
| 134 |
+
if key in seen:
|
| 135 |
+
continue
|
| 136 |
+
seen.add(key)
|
| 137 |
+
deduped.append(e)
|
| 138 |
+
out[source] = {
|
| 139 |
+
"rag_doc_id": h.get("doc_id"),
|
| 140 |
+
"title": h.get("title"),
|
| 141 |
+
"paragraph_excerpt": h.get("text", "")[:240]
|
| 142 |
+
+ ("β¦" if len(h.get("text", "")) > 240 else ""),
|
| 143 |
+
"n_entities": len(deduped),
|
| 144 |
+
"entities": [{"label": e.label, "text": e.text,
|
| 145 |
+
"score": round(e.score, 3)} for e in deduped],
|
| 146 |
+
}
|
| 147 |
+
return out
|
app/context/microtopo.py
ADDED
|
@@ -0,0 +1,274 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""LiDAR/DEM-derived micro-topography specialist.
|
| 2 |
+
|
| 3 |
+
Reads a window from a precomputed NYC-wide DEM (data/nyc_dem_30m.tif)
|
| 4 |
+
fetched from USGS 3DEP via py3dep. Computes per-address terrain numbers
|
| 5 |
+
that the static FEMA/DEP scenario maps don't expose.
|
| 6 |
+
|
| 7 |
+
Metrics (all derived from the same small AOI raster):
|
| 8 |
+
|
| 9 |
+
point_elev_m elevation at the address (m)
|
| 10 |
+
rel_elev_pct_750m percentile of point elev in a 750-m radius
|
| 11 |
+
rel_elev_pct_200m percentile of point elev in a 200-m radius
|
| 12 |
+
(block-scale "is this a bowl?")
|
| 13 |
+
basin_relief_m max-elev in 750-m AOI minus point elev
|
| 14 |
+
aoi_min_m, aoi_max_m for context
|
| 15 |
+
resolution_m
|
| 16 |
+
|
| 17 |
+
We deliberately stop at "shape-of-the-terrain" metrics rather than full
|
| 18 |
+
hydrology β depression-fill / D8 flow accumulation on a flat coastal
|
| 19 |
+
DEM are noisy and slow. Percentile + relief is what the reconciler
|
| 20 |
+
actually needs to write a useful sentence.
|
| 21 |
+
"""
|
| 22 |
+
from __future__ import annotations
|
| 23 |
+
|
| 24 |
+
import logging
|
| 25 |
+
import warnings
|
| 26 |
+
from dataclasses import dataclass
|
| 27 |
+
from pathlib import Path
|
| 28 |
+
|
| 29 |
+
import numpy as np
|
| 30 |
+
|
| 31 |
+
warnings.filterwarnings("ignore")
|
| 32 |
+
|
| 33 |
+
log = logging.getLogger("riprap.microtopo")
|
| 34 |
+
|
| 35 |
+
DOC_ID = "microtopo"
|
| 36 |
+
CITATION = "USGS 3DEP 30 m DEM (precomputed citywide GeoTIFF, WGS84)"
|
| 37 |
+
|
| 38 |
+
DATA_DIR = Path(__file__).resolve().parent.parent.parent / "data"
|
| 39 |
+
DEM_PATH = DATA_DIR / "nyc_dem_30m.tif"
|
| 40 |
+
TWI_PATH = DATA_DIR / "twi.tif"
|
| 41 |
+
HAND_PATH = DATA_DIR / "hand.tif"
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
@dataclass
|
| 45 |
+
class Microtopo:
|
| 46 |
+
point_elev_m: float
|
| 47 |
+
rel_elev_pct_750m: float # 0..100
|
| 48 |
+
rel_elev_pct_200m: float # 0..100
|
| 49 |
+
basin_relief_m: float
|
| 50 |
+
aoi_min_m: float
|
| 51 |
+
aoi_max_m: float
|
| 52 |
+
aoi_radius_m: int
|
| 53 |
+
resolution_m: int
|
| 54 |
+
# Hydrology indices computed on the same DEM (whitebox-workflows)
|
| 55 |
+
twi: float | None = None # Topographic Wetness Index, ln(SCA / tan(slope))
|
| 56 |
+
hand_m: float | None = None # Height Above Nearest Drainage (m)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def _percentile_in_window(arr: np.ndarray, iy: int, ix: int, point_val: float,
|
| 60 |
+
window_radius_cells: int) -> float:
|
| 61 |
+
H, W = arr.shape
|
| 62 |
+
y0 = max(0, iy - window_radius_cells)
|
| 63 |
+
y1 = min(H, iy + window_radius_cells + 1)
|
| 64 |
+
x0 = max(0, ix - window_radius_cells)
|
| 65 |
+
x1 = min(W, ix + window_radius_cells + 1)
|
| 66 |
+
sub = arr[y0:y1, x0:x1]
|
| 67 |
+
finite = sub[np.isfinite(sub)]
|
| 68 |
+
if finite.size == 0:
|
| 69 |
+
return float("nan")
|
| 70 |
+
return float((finite < point_val).sum()) / finite.size * 100.0
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
_DEM_CACHE: dict = {}
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def _read_full_raster(path: Path) -> tuple[np.ndarray | None, dict | None]:
|
| 77 |
+
import rasterio
|
| 78 |
+
if not path.exists():
|
| 79 |
+
return None, None
|
| 80 |
+
with rasterio.open(path) as ds:
|
| 81 |
+
arr = ds.read(1).astype("float32")
|
| 82 |
+
nodata = ds.nodata
|
| 83 |
+
meta = {"H": ds.height, "W": ds.width,
|
| 84 |
+
"transform": ds.transform, "crs": ds.crs, "nodata": nodata}
|
| 85 |
+
if nodata is not None:
|
| 86 |
+
arr = np.where(arr == nodata, np.nan, arr)
|
| 87 |
+
return arr, meta
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def _load_dem():
|
| 91 |
+
"""Read the precomputed NYC DEM + TWI + HAND rasters into memory.
|
| 92 |
+
|
| 93 |
+
All three are aligned (same grid, same transform). We hold them as
|
| 94 |
+
numpy arrays so per-query slicing is safe under threading.
|
| 95 |
+
"""
|
| 96 |
+
if "arr" in _DEM_CACHE:
|
| 97 |
+
return _DEM_CACHE
|
| 98 |
+
arr, meta = _read_full_raster(DEM_PATH)
|
| 99 |
+
if arr is None:
|
| 100 |
+
log.warning("microtopo DEM not found at %s β run scripts/fetch_nyc_dem.py", DEM_PATH)
|
| 101 |
+
return None
|
| 102 |
+
twi, _ = _read_full_raster(TWI_PATH)
|
| 103 |
+
hand, _ = _read_full_raster(HAND_PATH)
|
| 104 |
+
_DEM_CACHE.update({
|
| 105 |
+
"arr": arr, "H": meta["H"], "W": meta["W"],
|
| 106 |
+
"transform": meta["transform"], "crs": meta["crs"],
|
| 107 |
+
"twi": twi, "hand": hand,
|
| 108 |
+
})
|
| 109 |
+
note = []
|
| 110 |
+
if twi is not None: note.append(f"TWI {TWI_PATH.name}")
|
| 111 |
+
if hand is not None: note.append(f"HAND {HAND_PATH.name}")
|
| 112 |
+
log.info("microtopo: loaded NYC DEM %s (%dx%d, %s); aux: %s",
|
| 113 |
+
DEM_PATH.name, meta["H"], meta["W"], meta["crs"],
|
| 114 |
+
", ".join(note) if note else "(none β algorithmic only)")
|
| 115 |
+
return _DEM_CACHE
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def warm():
|
| 119 |
+
_load_dem()
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def _row_col(transform, lat: float, lon: float) -> tuple[int, int]:
|
| 123 |
+
"""Inverse-affine: WGS84 (lon,lat) -> raster (row, col).
|
| 124 |
+
Mirrors rasterio.transform.rowcol but without holding a dataset handle.
|
| 125 |
+
"""
|
| 126 |
+
# Diagonal affine (north-up raster): x = a*col + c, y = e*row + f.
|
| 127 |
+
a, c = transform.a, transform.c
|
| 128 |
+
e, f = transform.e, transform.f
|
| 129 |
+
col = int(round((lon - c) / a))
|
| 130 |
+
row = int(round((lat - f) / e))
|
| 131 |
+
return row, col
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def microtopo_at(lat: float, lon: float, radius_m: int = 750) -> Microtopo | None:
|
| 135 |
+
state = _load_dem()
|
| 136 |
+
if state is None:
|
| 137 |
+
return None
|
| 138 |
+
arr_full = state["arr"]
|
| 139 |
+
transform = state["transform"]
|
| 140 |
+
|
| 141 |
+
try:
|
| 142 |
+
row, col = _row_col(transform, lat, lon)
|
| 143 |
+
except Exception as e:
|
| 144 |
+
log.warning("microtopo index failed: %s", e)
|
| 145 |
+
return None
|
| 146 |
+
|
| 147 |
+
res_m = abs(transform.a) * 111_000.0 * np.cos(np.radians(lat))
|
| 148 |
+
cells_radius = max(2, int(np.ceil(radius_m / max(res_m, 1.0))))
|
| 149 |
+
|
| 150 |
+
H, W = state["H"], state["W"]
|
| 151 |
+
y0 = max(0, row - cells_radius); y1 = min(H, row + cells_radius + 1)
|
| 152 |
+
x0 = max(0, col - cells_radius); x1 = min(W, col + cells_radius + 1)
|
| 153 |
+
if y1 <= y0 or x1 <= x0:
|
| 154 |
+
return None
|
| 155 |
+
|
| 156 |
+
arr = arr_full[y0:y1, x0:x1].copy()
|
| 157 |
+
|
| 158 |
+
iy = row - y0
|
| 159 |
+
ix = col - x0
|
| 160 |
+
if not (0 <= iy < arr.shape[0] and 0 <= ix < arr.shape[1]):
|
| 161 |
+
return None
|
| 162 |
+
|
| 163 |
+
point_elev = float(arr[iy, ix])
|
| 164 |
+
if not np.isfinite(point_elev):
|
| 165 |
+
for r in range(1, 6):
|
| 166 |
+
ya, yb = max(0, iy - r), min(arr.shape[0], iy + r + 1)
|
| 167 |
+
xa, xb = max(0, ix - r), min(arr.shape[1], ix + r + 1)
|
| 168 |
+
sub = arr[ya:yb, xa:xb]
|
| 169 |
+
if np.isfinite(sub).any():
|
| 170 |
+
point_elev = float(np.nanmean(sub))
|
| 171 |
+
break
|
| 172 |
+
else:
|
| 173 |
+
return None
|
| 174 |
+
|
| 175 |
+
finite = arr[np.isfinite(arr)]
|
| 176 |
+
if finite.size == 0:
|
| 177 |
+
return None
|
| 178 |
+
aoi_min = float(finite.min())
|
| 179 |
+
aoi_max = float(finite.max())
|
| 180 |
+
|
| 181 |
+
pct_750 = float((finite < point_elev).sum()) / finite.size * 100.0
|
| 182 |
+
cells_200m = max(1, int(round(200 / max(res_m, 1.0))))
|
| 183 |
+
pct_200 = _percentile_in_window(arr, iy, ix, point_elev, cells_200m)
|
| 184 |
+
|
| 185 |
+
twi_arr = state.get("twi")
|
| 186 |
+
hand_arr = state.get("hand")
|
| 187 |
+
twi_v: float | None = None
|
| 188 |
+
hand_v: float | None = None
|
| 189 |
+
if twi_arr is not None and 0 <= row < H and 0 <= col < W:
|
| 190 |
+
v = float(twi_arr[row, col])
|
| 191 |
+
twi_v = round(v, 2) if np.isfinite(v) else None
|
| 192 |
+
if hand_arr is not None and 0 <= row < H and 0 <= col < W:
|
| 193 |
+
v = float(hand_arr[row, col])
|
| 194 |
+
hand_v = round(v, 2) if np.isfinite(v) else None
|
| 195 |
+
|
| 196 |
+
return Microtopo(
|
| 197 |
+
point_elev_m=round(point_elev, 2),
|
| 198 |
+
rel_elev_pct_750m=round(pct_750, 1),
|
| 199 |
+
rel_elev_pct_200m=round(pct_200, 1),
|
| 200 |
+
basin_relief_m=round(aoi_max - point_elev, 2),
|
| 201 |
+
aoi_min_m=round(aoi_min, 2),
|
| 202 |
+
aoi_max_m=round(aoi_max, 2),
|
| 203 |
+
aoi_radius_m=radius_m,
|
| 204 |
+
resolution_m=int(round(res_m)),
|
| 205 |
+
twi=twi_v,
|
| 206 |
+
hand_m=hand_v,
|
| 207 |
+
)
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
def microtopo_for_polygon(polygon, polygon_crs: str = "EPSG:4326") -> dict | None:
|
| 211 |
+
"""Polygon-mode aggregation: distributional summary of the DEM/HAND/TWI
|
| 212 |
+
rasters clipped to the polygon. Returns medians + fraction of cells
|
| 213 |
+
in flood-prone bands. Used for neighborhood-mode queries."""
|
| 214 |
+
state = _load_dem()
|
| 215 |
+
if state is None:
|
| 216 |
+
return None
|
| 217 |
+
try:
|
| 218 |
+
import rasterio
|
| 219 |
+
from rasterio.mask import mask as rio_mask
|
| 220 |
+
except Exception:
|
| 221 |
+
return None
|
| 222 |
+
import geopandas as gpd
|
| 223 |
+
|
| 224 |
+
poly = gpd.GeoDataFrame(geometry=[polygon], crs=polygon_crs).to_crs("EPSG:4326")
|
| 225 |
+
geom = [poly.iloc[0].geometry.__geo_interface__]
|
| 226 |
+
|
| 227 |
+
def _stats(path: Path) -> dict | None:
|
| 228 |
+
if not path.exists():
|
| 229 |
+
return None
|
| 230 |
+
try:
|
| 231 |
+
with rasterio.open(path) as src:
|
| 232 |
+
clipped, _ = rio_mask(src, geom, crop=True, filled=False)
|
| 233 |
+
arr = clipped[0]
|
| 234 |
+
vals = arr.compressed() if hasattr(arr, "compressed") else arr.flatten()
|
| 235 |
+
vals = vals[np.isfinite(vals)]
|
| 236 |
+
if vals.size == 0:
|
| 237 |
+
return None
|
| 238 |
+
return {
|
| 239 |
+
"n_cells": int(vals.size),
|
| 240 |
+
"min": float(np.min(vals)),
|
| 241 |
+
"median": float(np.median(vals)),
|
| 242 |
+
"p10": float(np.percentile(vals, 10)),
|
| 243 |
+
"p90": float(np.percentile(vals, 90)),
|
| 244 |
+
"max": float(np.max(vals)),
|
| 245 |
+
"raw": vals,
|
| 246 |
+
}
|
| 247 |
+
except Exception as e:
|
| 248 |
+
log.warning("polygon raster mask failed for %s: %r", path.name, e)
|
| 249 |
+
return None
|
| 250 |
+
|
| 251 |
+
elev = _stats(DEM_PATH)
|
| 252 |
+
hand = _stats(HAND_PATH)
|
| 253 |
+
twi = _stats(TWI_PATH)
|
| 254 |
+
if elev is None:
|
| 255 |
+
return None
|
| 256 |
+
|
| 257 |
+
# Fraction of polygon cells in canonical flood-prone bands
|
| 258 |
+
frac_hand_lt1 = (
|
| 259 |
+
round(float((hand["raw"] < 1.0).mean()), 4) if hand else None
|
| 260 |
+
)
|
| 261 |
+
frac_twi_gt10 = (
|
| 262 |
+
round(float((twi["raw"] > 10.0).mean()), 4) if twi else None
|
| 263 |
+
)
|
| 264 |
+
return {
|
| 265 |
+
"n_cells": elev["n_cells"],
|
| 266 |
+
"elev_min_m": round(elev["min"], 2),
|
| 267 |
+
"elev_median_m": round(elev["median"], 2),
|
| 268 |
+
"elev_p10_m": round(elev["p10"], 2),
|
| 269 |
+
"elev_max_m": round(elev["max"], 2),
|
| 270 |
+
"hand_median_m": round(hand["median"], 2) if hand else None,
|
| 271 |
+
"twi_median": round(twi["median"], 2) if twi else None,
|
| 272 |
+
"frac_hand_lt1": frac_hand_lt1,
|
| 273 |
+
"frac_twi_gt10": frac_twi_gt10,
|
| 274 |
+
}
|
app/context/noaa_tides.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""NOAA CO-OPS Tides & Currents β live coastal water level.
|
| 2 |
+
|
| 3 |
+
api.tidesandcurrents.noaa.gov, no auth, 6-min cadence.
|
| 4 |
+
|
| 5 |
+
We pick the nearest of three NYC-region stations to the queried address:
|
| 6 |
+
- 8518750 The Battery, NY
|
| 7 |
+
- 8516945 Kings Point, NY (Long Island Sound entrance)
|
| 8 |
+
- 8531680 Sandy Hook, NJ (NY Harbor approach)
|
| 9 |
+
|
| 10 |
+
The verified-water-level API returns instantaneous water elevation
|
| 11 |
+
relative to MLLW (Mean Lower Low Water β the local tidal datum). To
|
| 12 |
+
distinguish "high tide" from "storm surge" we also fetch the published
|
| 13 |
+
predicted tide and report the residual.
|
| 14 |
+
"""
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
from dataclasses import dataclass
|
| 18 |
+
from math import asin, cos, radians, sin, sqrt
|
| 19 |
+
|
| 20 |
+
import httpx
|
| 21 |
+
|
| 22 |
+
DOC_ID = "noaa_tides"
|
| 23 |
+
CITATION = "NOAA CO-OPS Tides & Currents (api.tidesandcurrents.noaa.gov)"
|
| 24 |
+
URL = "https://api.tidesandcurrents.noaa.gov/api/prod/datagetter"
|
| 25 |
+
|
| 26 |
+
STATIONS = [
|
| 27 |
+
# (id, name, lat, lon)
|
| 28 |
+
# NYC harbor + Long Island Sound
|
| 29 |
+
("8518750", "The Battery, NY", 40.7006, -74.0142),
|
| 30 |
+
("8516945", "Kings Point, NY", 40.8103, -73.7649),
|
| 31 |
+
("8531680", "Sandy Hook, NJ", 40.4669, -74.0094),
|
| 32 |
+
# Hudson tidal corridor (head-of-tide is Troy / Albany; Hudson is tidal
|
| 33 |
+
# all the way up to the Federal Lock at Troy)
|
| 34 |
+
("8518995", "Albany, NY (Hudson)", 42.6469, -73.7464),
|
| 35 |
+
("8518962", "Turkey Point Hudson, NY", 41.7569, -73.9433),
|
| 36 |
+
("8519483", "West Point, NY", 41.3845, -73.9536),
|
| 37 |
+
]
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
@dataclass
|
| 41 |
+
class TideReading:
|
| 42 |
+
station_id: str
|
| 43 |
+
station_name: str
|
| 44 |
+
distance_km: float
|
| 45 |
+
observed_ft: float | None # current water level above MLLW
|
| 46 |
+
predicted_ft: float | None # astronomical prediction at same instant
|
| 47 |
+
residual_ft: float | None # observed - predicted (β storm surge)
|
| 48 |
+
obs_time: str | None
|
| 49 |
+
error: str | None = None
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _haversine_km(lat1, lon1, lat2, lon2) -> float:
|
| 53 |
+
R = 6371.0
|
| 54 |
+
p1, p2 = radians(lat1), radians(lat2)
|
| 55 |
+
dp = radians(lat2 - lat1); dl = radians(lon2 - lon1)
|
| 56 |
+
a = sin(dp/2)**2 + cos(p1)*cos(p2)*sin(dl/2)**2
|
| 57 |
+
return 2 * R * asin(sqrt(a))
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def _nearest_station(lat: float, lon: float):
|
| 61 |
+
return min(STATIONS, key=lambda s: _haversine_km(lat, lon, s[2], s[3]))
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def _fetch(station_id: str, product: str) -> dict:
|
| 65 |
+
r = httpx.get(URL, params={
|
| 66 |
+
"date": "latest", "station": station_id, "product": product,
|
| 67 |
+
"datum": "MLLW", "units": "english", "time_zone": "lst_ldt",
|
| 68 |
+
"format": "json",
|
| 69 |
+
}, timeout=8.0)
|
| 70 |
+
r.raise_for_status()
|
| 71 |
+
return r.json()
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def reading_at(lat: float, lon: float) -> TideReading:
|
| 75 |
+
sid, name, slat, slon = _nearest_station(lat, lon)
|
| 76 |
+
dist_km = round(_haversine_km(lat, lon, slat, slon), 1)
|
| 77 |
+
out = TideReading(station_id=sid, station_name=name, distance_km=dist_km,
|
| 78 |
+
observed_ft=None, predicted_ft=None, residual_ft=None,
|
| 79 |
+
obs_time=None)
|
| 80 |
+
try:
|
| 81 |
+
obs = _fetch(sid, "water_level").get("data") or []
|
| 82 |
+
pred = _fetch(sid, "predictions").get("predictions") or []
|
| 83 |
+
if obs:
|
| 84 |
+
out.observed_ft = round(float(obs[0]["v"]), 2)
|
| 85 |
+
out.obs_time = obs[0].get("t")
|
| 86 |
+
if pred:
|
| 87 |
+
out.predicted_ft = round(float(pred[0]["v"]), 2)
|
| 88 |
+
if out.observed_ft is not None and out.predicted_ft is not None:
|
| 89 |
+
out.residual_ft = round(out.observed_ft - out.predicted_ft, 2)
|
| 90 |
+
except Exception as e:
|
| 91 |
+
out.error = str(e)
|
| 92 |
+
return out
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def summary_for_point(lat: float, lon: float) -> dict:
|
| 96 |
+
r = reading_at(lat, lon)
|
| 97 |
+
# Look up station coords for the map marker.
|
| 98 |
+
sta = next((s for s in STATIONS if s[0] == r.station_id), None)
|
| 99 |
+
return {
|
| 100 |
+
"station_id": r.station_id,
|
| 101 |
+
"station_name": r.station_name,
|
| 102 |
+
"station_lat": sta[2] if sta else None,
|
| 103 |
+
"station_lon": sta[3] if sta else None,
|
| 104 |
+
"distance_km": r.distance_km,
|
| 105 |
+
"observed_ft_mllw": r.observed_ft,
|
| 106 |
+
"predicted_ft_mllw": r.predicted_ft,
|
| 107 |
+
"residual_ft": r.residual_ft,
|
| 108 |
+
"obs_time": r.obs_time,
|
| 109 |
+
"error": r.error,
|
| 110 |
+
}
|
app/context/npcc4_slr.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""NPCC4 sea-level rise projections for NYC (static lookup).
|
| 2 |
+
|
| 3 |
+
Source: New York City Panel on Climate Change 4th Assessment (2024),
|
| 4 |
+
Chapter 3, Table 3.2 β sea-level rise relative to 2000β2004 baseline,
|
| 5 |
+
Battery Tide Gauge (NOAA 8518750), primary NYC harbor reference.
|
| 6 |
+
|
| 7 |
+
Values are in inches above the 2000β2004 mean. The NPCC4 uses a
|
| 8 |
+
probabilistic framework across RCP/SSP scenarios; the table excerpted
|
| 9 |
+
here represents the "likely range" (10thβ90th) plus the high-end
|
| 10 |
+
"extreme" scenario (99th).
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
DOC_ID = "npcc4_slr"
|
| 14 |
+
CITATION = (
|
| 15 |
+
"New York City Panel on Climate Change 4th Assessment (NPCC4 2024), "
|
| 16 |
+
"Chapter 3 β Sea Level Rise, Table 3.2. "
|
| 17 |
+
"Published by the New York Academy of Sciences. "
|
| 18 |
+
"Reference gauge: NOAA Battery (8518750), baseline 2000β2004."
|
| 19 |
+
)
|
| 20 |
+
|
| 21 |
+
# Sea-level rise projections in INCHES above the 2000β2004 baseline,
|
| 22 |
+
# Battery Tide Gauge. Percentiles: 10th (low), 50th (mid), 90th (high),
|
| 23 |
+
# 99th (extreme). All values from NPCC4 (2024) Ch. 3 Table 3.2.
|
| 24 |
+
_TABLE_IN = {
|
| 25 |
+
2050: {10: 8, 50: 15, 90: 29, 99: 40},
|
| 26 |
+
2100: {10: 13, 50: 31, 90: 65, 99: 96},
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _in_to_m(inches: float) -> float:
|
| 31 |
+
return round(inches * 0.0254, 2)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def get_projections() -> dict:
|
| 35 |
+
"""Return NPCC4 SLR projection dict, always available (static table)."""
|
| 36 |
+
result: dict = {"available": True, "baseline": "2000β2004", "gauge": "NOAA Battery (8518750)"}
|
| 37 |
+
for year, pcts in _TABLE_IN.items():
|
| 38 |
+
result[str(year)] = {
|
| 39 |
+
str(pct): {"in": v, "m": _in_to_m(v)}
|
| 40 |
+
for pct, v in pcts.items()
|
| 41 |
+
}
|
| 42 |
+
return result
|
app/context/nws_alerts.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""NWS API β active alerts at a point.
|
| 2 |
+
|
| 3 |
+
api.weather.gov/alerts/active?point={lat},{lon}, no auth, JSON.
|
| 4 |
+
A User-Agent header is required (NWS rate-limits anonymous traffic).
|
| 5 |
+
|
| 6 |
+
We surface only flood-relevant categories so the doc the reconciler
|
| 7 |
+
sees is short and on-topic.
|
| 8 |
+
"""
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
from typing import Any
|
| 12 |
+
|
| 13 |
+
import httpx
|
| 14 |
+
|
| 15 |
+
DOC_ID = "nws_alerts"
|
| 16 |
+
CITATION = "NWS public alert API (api.weather.gov/alerts)"
|
| 17 |
+
|
| 18 |
+
USER_AGENT = "Riprap-NYC/0.1 (civic-flood-tool; +https://huggingface.co/spaces/msradam/riprap-nyc)"
|
| 19 |
+
|
| 20 |
+
_FLOOD_EVENT_KEYWORDS = (
|
| 21 |
+
"flood", "flash flood", "coastal flood", "high surf", "storm surge",
|
| 22 |
+
"hurricane", "tropical storm", "tornado warning", # high-impact context
|
| 23 |
+
"rip current",
|
| 24 |
+
)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def _is_flood_relevant(event_name: str) -> bool:
|
| 28 |
+
e = (event_name or "").lower()
|
| 29 |
+
return any(k in e for k in _FLOOD_EVENT_KEYWORDS)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def alerts_at(lat: float, lon: float) -> list[dict[str, Any]]:
|
| 33 |
+
r = httpx.get(
|
| 34 |
+
"https://api.weather.gov/alerts/active",
|
| 35 |
+
params={"point": f"{lat:.4f},{lon:.4f}"},
|
| 36 |
+
headers={"User-Agent": USER_AGENT, "Accept": "application/geo+json"},
|
| 37 |
+
timeout=8.0,
|
| 38 |
+
)
|
| 39 |
+
r.raise_for_status()
|
| 40 |
+
out = []
|
| 41 |
+
for f in r.json().get("features", []):
|
| 42 |
+
p = f.get("properties", {}) or {}
|
| 43 |
+
event = p.get("event") or ""
|
| 44 |
+
if not _is_flood_relevant(event):
|
| 45 |
+
continue
|
| 46 |
+
out.append({
|
| 47 |
+
"id": p.get("id"),
|
| 48 |
+
"event": event,
|
| 49 |
+
"severity": p.get("severity"),
|
| 50 |
+
"urgency": p.get("urgency"),
|
| 51 |
+
"certainty": p.get("certainty"),
|
| 52 |
+
"headline": p.get("headline"),
|
| 53 |
+
"sent": p.get("sent"),
|
| 54 |
+
"effective": p.get("effective"),
|
| 55 |
+
"expires": p.get("expires"),
|
| 56 |
+
"sender_name": p.get("senderName"),
|
| 57 |
+
"areaDesc": p.get("areaDesc"),
|
| 58 |
+
})
|
| 59 |
+
return out
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def summary_for_point(lat: float, lon: float) -> dict:
|
| 63 |
+
try:
|
| 64 |
+
active = alerts_at(lat, lon)
|
| 65 |
+
except Exception as e:
|
| 66 |
+
return {"n_active": 0, "alerts": [], "error": str(e)}
|
| 67 |
+
return {
|
| 68 |
+
"n_active": len(active),
|
| 69 |
+
"alerts": active,
|
| 70 |
+
"error": None,
|
| 71 |
+
}
|
app/context/nws_obs.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""NWS station observations β latest hourly METAR for the nearest NYC airport.
|
| 2 |
+
|
| 3 |
+
api.weather.gov/stations/{id}/observations/latest.
|
| 4 |
+
|
| 5 |
+
Five NYC-region ASOS stations cover the city; we pick the nearest.
|
| 6 |
+
Most useful field for flood context is hourly precipitation (the
|
| 7 |
+
`precipitationLastHour` quantity, mm). The latest observation is
|
| 8 |
+
typically <60 min old.
|
| 9 |
+
"""
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
from dataclasses import dataclass
|
| 13 |
+
from math import asin, cos, radians, sin, sqrt
|
| 14 |
+
|
| 15 |
+
import httpx
|
| 16 |
+
|
| 17 |
+
DOC_ID = "nws_obs"
|
| 18 |
+
CITATION = "NWS station observations API (api.weather.gov/stations)"
|
| 19 |
+
|
| 20 |
+
USER_AGENT = "Riprap-NYC/0.1 (civic-flood-tool; +https://huggingface.co/spaces/msradam/riprap-nyc)"
|
| 21 |
+
|
| 22 |
+
# NYC + Hudson Corridor ASOS stations. Picker is haversine-nearest, so adding
|
| 23 |
+
# upstate stations enables Albany / Poughkeepsie / Newburgh queries without
|
| 24 |
+
# breaking NYC behaviour (NYC stations stay closer for NYC lat/lon).
|
| 25 |
+
STATIONS = [
|
| 26 |
+
# NYC region
|
| 27 |
+
("KNYC", "Central Park, NY", 40.7794, -73.9692),
|
| 28 |
+
("KLGA", "LaGuardia Airport, NY", 40.7794, -73.8800),
|
| 29 |
+
("KJFK", "JFK Airport, NY", 40.6413, -73.7781),
|
| 30 |
+
("KEWR", "Newark Liberty, NJ", 40.6925, -74.1687),
|
| 31 |
+
("KFRG", "Republic Farmingdale, NY", 40.7288, -73.4134),
|
| 32 |
+
# Hudson Corridor (south β north)
|
| 33 |
+
("KHPN", "White Plains, NY", 41.0670, -73.7076),
|
| 34 |
+
("KSWF", "Newburgh-Stewart, NY", 41.5042, -74.1048),
|
| 35 |
+
("KPOU", "Poughkeepsie, NY", 41.6262, -73.8842),
|
| 36 |
+
("KALB", "Albany Intl, NY", 42.7475, -73.8025),
|
| 37 |
+
]
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
@dataclass
|
| 41 |
+
class Obs:
|
| 42 |
+
station_id: str
|
| 43 |
+
station_name: str
|
| 44 |
+
distance_km: float
|
| 45 |
+
obs_time: str | None
|
| 46 |
+
temp_c: float | None
|
| 47 |
+
precip_last_hour_mm: float | None
|
| 48 |
+
precip_last_3h_mm: float | None
|
| 49 |
+
precip_last_6h_mm: float | None
|
| 50 |
+
error: str | None = None
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def _haversine_km(lat1, lon1, lat2, lon2) -> float:
|
| 54 |
+
R = 6371.0
|
| 55 |
+
p1, p2 = radians(lat1), radians(lat2)
|
| 56 |
+
dp = radians(lat2 - lat1); dl = radians(lon2 - lon1)
|
| 57 |
+
a = sin(dp/2)**2 + cos(p1)*cos(p2)*sin(dl/2)**2
|
| 58 |
+
return 2 * R * asin(sqrt(a))
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _val_mm(props, key) -> float | None:
|
| 62 |
+
"""NWS returns {value: ..., unitCode: 'wmoUnit:mm'} per quantity. Convert
|
| 63 |
+
to mm; if value is null, return None."""
|
| 64 |
+
q = (props or {}).get(key) or {}
|
| 65 |
+
v = q.get("value")
|
| 66 |
+
if v is None:
|
| 67 |
+
return None
|
| 68 |
+
return round(float(v), 2)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def obs_at(lat: float, lon: float) -> Obs:
|
| 72 |
+
sid, name, slat, slon = min(STATIONS, key=lambda s: _haversine_km(lat, lon, s[2], s[3]))
|
| 73 |
+
dist_km = round(_haversine_km(lat, lon, slat, slon), 1)
|
| 74 |
+
out = Obs(station_id=sid, station_name=name, distance_km=dist_km,
|
| 75 |
+
obs_time=None, temp_c=None,
|
| 76 |
+
precip_last_hour_mm=None, precip_last_3h_mm=None,
|
| 77 |
+
precip_last_6h_mm=None)
|
| 78 |
+
try:
|
| 79 |
+
r = httpx.get(
|
| 80 |
+
f"https://api.weather.gov/stations/{sid}/observations/latest",
|
| 81 |
+
headers={"User-Agent": USER_AGENT, "Accept": "application/geo+json"},
|
| 82 |
+
timeout=8.0,
|
| 83 |
+
)
|
| 84 |
+
r.raise_for_status()
|
| 85 |
+
p = r.json().get("properties", {}) or {}
|
| 86 |
+
out.obs_time = p.get("timestamp")
|
| 87 |
+
out.temp_c = _val_mm(p, "temperature")
|
| 88 |
+
out.precip_last_hour_mm = _val_mm(p, "precipitationLastHour")
|
| 89 |
+
out.precip_last_3h_mm = _val_mm(p, "precipitationLast3Hours")
|
| 90 |
+
out.precip_last_6h_mm = _val_mm(p, "precipitationLast6Hours")
|
| 91 |
+
except Exception as e:
|
| 92 |
+
out.error = str(e)
|
| 93 |
+
return out
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def summary_for_point(lat: float, lon: float) -> dict:
|
| 97 |
+
o = obs_at(lat, lon)
|
| 98 |
+
return {
|
| 99 |
+
"station_id": o.station_id,
|
| 100 |
+
"station_name": o.station_name,
|
| 101 |
+
"distance_km": o.distance_km,
|
| 102 |
+
"obs_time": o.obs_time,
|
| 103 |
+
"temp_c": o.temp_c,
|
| 104 |
+
"precip_last_hour_mm": o.precip_last_hour_mm,
|
| 105 |
+
"precip_last_3h_mm": o.precip_last_3h_mm,
|
| 106 |
+
"precip_last_6h_mm": o.precip_last_6h_mm,
|
| 107 |
+
"error": o.error,
|
| 108 |
+
}
|
app/context/nyc311.py
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""NYC 311 β flood-related complaints around a point.
|
| 2 |
+
|
| 3 |
+
Live dataset: erm2-nwe9. Filter by descriptor (the flood signal is in
|
| 4 |
+
descriptor, not complaint_type) within a buffer.
|
| 5 |
+
"""
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
from collections import Counter
|
| 9 |
+
from dataclasses import dataclass
|
| 10 |
+
from datetime import datetime, timedelta, timezone
|
| 11 |
+
|
| 12 |
+
import httpx
|
| 13 |
+
|
| 14 |
+
URL = "https://data.cityofnewyork.us/resource/erm2-nwe9.json"
|
| 15 |
+
DOC_ID = "nyc311"
|
| 16 |
+
CITATION = "NYC 311 service requests (Socrata erm2-nwe9, 2010-present)"
|
| 17 |
+
|
| 18 |
+
FLOOD_DESCRIPTORS = [
|
| 19 |
+
"Street Flooding (SJ)",
|
| 20 |
+
"Sewer Backup (Use Comments) (SA)",
|
| 21 |
+
"Catch Basin Clogged/Flooding (Use Comments) (SC)",
|
| 22 |
+
"Highway Flooding (SH)",
|
| 23 |
+
"Manhole Overflow (Use Comments) (SA1)",
|
| 24 |
+
"Flooding on Street",
|
| 25 |
+
"RAIN GARDEN FLOODING (SRGFLD)",
|
| 26 |
+
]
|
| 27 |
+
|
| 28 |
+
_DESC_CLAUSE = "(" + " OR ".join(f"descriptor='{d}'" for d in FLOOD_DESCRIPTORS) + ")"
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
@dataclass
|
| 32 |
+
class Complaint:
|
| 33 |
+
unique_key: str
|
| 34 |
+
descriptor: str
|
| 35 |
+
created_date: str
|
| 36 |
+
address: str | None
|
| 37 |
+
status: str | None
|
| 38 |
+
lat: float | None = None
|
| 39 |
+
lon: float | None = None
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def complaints_near(lat: float, lon: float, radius_m: float = 200,
|
| 43 |
+
since: datetime | None = None,
|
| 44 |
+
limit: int = 1000) -> list[Complaint]:
|
| 45 |
+
where = f"{_DESC_CLAUSE} AND within_circle(location, {lat}, {lon}, {radius_m})"
|
| 46 |
+
if since:
|
| 47 |
+
# Socrata floating-timestamp: drop tz suffix
|
| 48 |
+
ts = since.replace(tzinfo=None).isoformat(timespec="seconds")
|
| 49 |
+
where += f" AND created_date >= '{ts}'"
|
| 50 |
+
r = httpx.get(URL, params={
|
| 51 |
+
"$select": "unique_key, descriptor, created_date, incident_address, "
|
| 52 |
+
"status, latitude, longitude",
|
| 53 |
+
"$where": where,
|
| 54 |
+
"$order": "created_date desc",
|
| 55 |
+
"$limit": str(limit),
|
| 56 |
+
}, timeout=30)
|
| 57 |
+
r.raise_for_status()
|
| 58 |
+
out = []
|
| 59 |
+
for row in r.json():
|
| 60 |
+
lat = row.get("latitude")
|
| 61 |
+
lon = row.get("longitude")
|
| 62 |
+
try:
|
| 63 |
+
lat = float(lat) if lat is not None else None
|
| 64 |
+
lon = float(lon) if lon is not None else None
|
| 65 |
+
except Exception:
|
| 66 |
+
lat, lon = None, None
|
| 67 |
+
out.append(Complaint(
|
| 68 |
+
unique_key=row.get("unique_key", ""),
|
| 69 |
+
descriptor=row.get("descriptor", ""),
|
| 70 |
+
created_date=row.get("created_date", ""),
|
| 71 |
+
address=row.get("incident_address"),
|
| 72 |
+
status=row.get("status"),
|
| 73 |
+
lat=lat, lon=lon,
|
| 74 |
+
))
|
| 75 |
+
return out
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def summary_for_point(lat: float, lon: float, radius_m: float = 200,
|
| 79 |
+
years: int = 5) -> dict:
|
| 80 |
+
since = datetime.now(timezone.utc) - timedelta(days=365 * years)
|
| 81 |
+
cs = complaints_near(lat, lon, radius_m, since=since, limit=2000)
|
| 82 |
+
return _summarize(cs, years=years, radius_m=radius_m)
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def complaints_in_polygon(polygon, polygon_crs: str = "EPSG:4326",
|
| 86 |
+
since: datetime | None = None,
|
| 87 |
+
limit: int = 5000,
|
| 88 |
+
simplify_tolerance: float = 0.0005) -> list[Complaint]:
|
| 89 |
+
"""Pull flood-related complaints inside an arbitrary polygon via
|
| 90 |
+
Socrata's `within_polygon(location, 'MULTIPOLYGON(...)')` predicate.
|
| 91 |
+
|
| 92 |
+
NYC NTA polygons can have thousands of vertices and exceed Socrata's
|
| 93 |
+
URL length limit (414). We simplify in EPSG:4326 with a default
|
| 94 |
+
~50 m tolerance, which collapses vertex count ~10-20Γ without
|
| 95 |
+
materially changing the contained-points result.
|
| 96 |
+
|
| 97 |
+
Polygon must be EPSG:4326 (lat/lon) for the Socrata query.
|
| 98 |
+
"""
|
| 99 |
+
import geopandas as gpd
|
| 100 |
+
g = gpd.GeoDataFrame(geometry=[polygon], crs=polygon_crs).to_crs("EPSG:4326")
|
| 101 |
+
geom = g.iloc[0].geometry.simplify(simplify_tolerance, preserve_topology=True)
|
| 102 |
+
wkt = geom.wkt
|
| 103 |
+
where = f"{_DESC_CLAUSE} AND within_polygon(location, '{wkt}')"
|
| 104 |
+
if since:
|
| 105 |
+
ts = since.replace(tzinfo=None).isoformat(timespec="seconds")
|
| 106 |
+
where += f" AND created_date >= '{ts}'"
|
| 107 |
+
r = httpx.get(URL, params={
|
| 108 |
+
"$select": "unique_key, descriptor, created_date, incident_address, status",
|
| 109 |
+
"$where": where,
|
| 110 |
+
"$order": "created_date desc",
|
| 111 |
+
"$limit": str(limit),
|
| 112 |
+
}, timeout=60)
|
| 113 |
+
r.raise_for_status()
|
| 114 |
+
return [
|
| 115 |
+
Complaint(
|
| 116 |
+
unique_key=row.get("unique_key", ""),
|
| 117 |
+
descriptor=row.get("descriptor", ""),
|
| 118 |
+
created_date=row.get("created_date", ""),
|
| 119 |
+
address=row.get("incident_address"),
|
| 120 |
+
status=row.get("status"),
|
| 121 |
+
)
|
| 122 |
+
for row in r.json()
|
| 123 |
+
]
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def summary_for_polygon(polygon, polygon_crs: str = "EPSG:4326",
|
| 127 |
+
years: int = 5) -> dict:
|
| 128 |
+
"""Polygon-mode aggregation: counts of flood-related 311 complaints
|
| 129 |
+
inside the polygon over the trailing window."""
|
| 130 |
+
since = datetime.now(timezone.utc) - timedelta(days=365 * years)
|
| 131 |
+
cs = complaints_in_polygon(polygon, polygon_crs=polygon_crs, since=since)
|
| 132 |
+
return _summarize(cs, years=years, radius_m=None)
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def _summarize(cs: list[Complaint], years: int, radius_m: float | None) -> dict:
|
| 136 |
+
by_year: Counter = Counter(c.created_date[:4] for c in cs if c.created_date)
|
| 137 |
+
by_descriptor: Counter = Counter(c.descriptor for c in cs)
|
| 138 |
+
# Cap at 60 most-recent points for the map layer β keeps the SSE
|
| 139 |
+
# payload small while still showing meaningful clustering.
|
| 140 |
+
points = [
|
| 141 |
+
{"lat": c.lat, "lon": c.lon,
|
| 142 |
+
"descriptor": c.descriptor,
|
| 143 |
+
"date": c.created_date[:10],
|
| 144 |
+
"address": c.address}
|
| 145 |
+
for c in cs[:60]
|
| 146 |
+
if c.lat is not None and c.lon is not None
|
| 147 |
+
]
|
| 148 |
+
return {
|
| 149 |
+
"n": len(cs),
|
| 150 |
+
"radius_m": radius_m,
|
| 151 |
+
"years": years,
|
| 152 |
+
"by_year": dict(sorted(by_year.items())),
|
| 153 |
+
"by_descriptor": dict(by_descriptor.most_common(6)),
|
| 154 |
+
"most_recent": [
|
| 155 |
+
{"date": c.created_date[:10],
|
| 156 |
+
"descriptor": c.descriptor,
|
| 157 |
+
"address": c.address}
|
| 158 |
+
for c in cs[:5]
|
| 159 |
+
],
|
| 160 |
+
"points": points,
|
| 161 |
+
}
|
app/context/terramind_nyc.py
ADDED
|
@@ -0,0 +1,485 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""TerraMind-NYC adapters β LULC and Buildings inference for NYC chips.
|
| 2 |
+
|
| 3 |
+
Wraps the Apache-2.0 [`msradam/TerraMind-NYC-Adapters`](https://huggingface.co/msradam/TerraMind-NYC-Adapters)
|
| 4 |
+
LoRA family fine-tuned on NYC EO chips (Sentinel-2 L2A + Sentinel-1 RTC
|
| 5 |
+
+ Copernicus DEM, temporal stack of 4) on AMD MI300X via AMD Developer
|
| 6 |
+
Cloud. Exposes two specialist entry points:
|
| 7 |
+
|
| 8 |
+
lulc(s2l2a, s1rtc, dem) -> 5-class macro NYC LULC mask
|
| 9 |
+
buildings(s2l2a, s1rtc, dem) -> binary NYC building footprint mask
|
| 10 |
+
|
| 11 |
+
The base TerraMind 1.0 weights are downloaded by terratorch on first
|
| 12 |
+
call; the LoRA adapter + UNet decoder weights come from the HF repo and
|
| 13 |
+
are cached to `~/.cache/huggingface/hub`.
|
| 14 |
+
|
| 15 |
+
CHIP-SIZE TRAP. TerraMind's positional embeddings don't generalise off
|
| 16 |
+
its training resolution (224Γ224). Calling `task.model({...})` on a
|
| 17 |
+
chip β 224Γ224 produces silent garbage. We therefore wrap inference
|
| 18 |
+
with `terratorch.tasks.tiled_inference.tiled_inference`, which slides
|
| 19 |
+
a 224Γ224 crop window across the chip and stitches per-window logits.
|
| 20 |
+
This matches the patch in
|
| 21 |
+
`experiments/18_terramind_nyc_lora/shared/inference_ensemble.py` that
|
| 22 |
+
the plan flags as required for production.
|
| 23 |
+
|
| 24 |
+
Gated by RIPRAP_TERRAMIND_NYC_ENABLE β deployments without the deps
|
| 25 |
+
installed (HF Spaces' Py3.10 cone, plain Ollama dev VMs) silently no-op
|
| 26 |
+
through the same skipped-result shape every other heavy specialist
|
| 27 |
+
emits.
|
| 28 |
+
|
| 29 |
+
This module does NOT fetch its own S2/S1/DEM chips. C4 wires it into
|
| 30 |
+
the FSM with a shared chip cache so the LULC and Buildings calls
|
| 31 |
+
don't each refetch ~150 MB of imagery.
|
| 32 |
+
"""
|
| 33 |
+
from __future__ import annotations
|
| 34 |
+
|
| 35 |
+
import logging
|
| 36 |
+
import os
|
| 37 |
+
import threading
|
| 38 |
+
import time
|
| 39 |
+
from typing import Any
|
| 40 |
+
|
| 41 |
+
log = logging.getLogger("riprap.terramind_nyc")
|
| 42 |
+
|
| 43 |
+
ENABLE = os.environ.get("RIPRAP_TERRAMIND_NYC_ENABLE", "1").lower() in ("1", "true", "yes")
|
| 44 |
+
DEVICE = os.environ.get("RIPRAP_TERRAMIND_NYC_DEVICE", "cpu")
|
| 45 |
+
ADAPTERS_REPO = "msradam/TerraMind-NYC-Adapters"
|
| 46 |
+
|
| 47 |
+
# Per-task config knobs the HF README's quick-start fixes for these
|
| 48 |
+
# adapters. Mirrored from experiments/18_terramind_nyc_lora/adapters/*/
|
| 49 |
+
# config.yaml so a single source of truth lives next to the inference
|
| 50 |
+
# code rather than being scraped from YAML at runtime.
|
| 51 |
+
ADAPTER_SPECS: dict[str, dict[str, Any]] = {
|
| 52 |
+
"lulc": {
|
| 53 |
+
"subdir": "lulc_nyc",
|
| 54 |
+
"num_classes": 5,
|
| 55 |
+
"class_labels": [
|
| 56 |
+
"Trees / vegetation",
|
| 57 |
+
"Cropland",
|
| 58 |
+
"Built / impervious",
|
| 59 |
+
"Bare ground",
|
| 60 |
+
"Water",
|
| 61 |
+
],
|
| 62 |
+
},
|
| 63 |
+
"buildings": {
|
| 64 |
+
"subdir": "buildings_nyc",
|
| 65 |
+
"num_classes": 2,
|
| 66 |
+
# The decoder emits class 0 = background, class 1 = building.
|
| 67 |
+
"class_labels": ["Background", "Building footprint"],
|
| 68 |
+
},
|
| 69 |
+
}
|
| 70 |
+
|
| 71 |
+
# Tile-window size β TerraMind's training resolution. Stride < window
|
| 72 |
+
# yields overlap (smooths seams from window-boundary classification
|
| 73 |
+
# noise); 96 px overlap matches the experiments/18 ensemble.
|
| 74 |
+
TILE_SIZE = 224
|
| 75 |
+
TILE_STRIDE = 128
|
| 76 |
+
|
| 77 |
+
# One-shot lazy-init guards. The base TerraMind weights are heavy
|
| 78 |
+
# (~1.6 GB) and we want to load them once across LULC and Buildings.
|
| 79 |
+
_INIT_LOCK = threading.Lock()
|
| 80 |
+
_BASE_LOADED = False
|
| 81 |
+
_ADAPTERS: dict[str, Any] = {} # name -> built terratorch task on DEVICE
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def _has_required_deps() -> tuple[bool, str | None]:
|
| 85 |
+
"""Probe the heavy-EO deps. Same shape as prithvi_live's check β
|
| 86 |
+
a missing dep (terratorch / peft / safetensors / hf_hub) returns a
|
| 87 |
+
clean `skipped: deps_unavailable` outcome instead of a noisy
|
| 88 |
+
ModuleNotFoundError in the trace.
|
| 89 |
+
|
| 90 |
+
On the HF Space, terratorch's import chain itself can raise
|
| 91 |
+
RuntimeError("operator torchvision::nms does not exist") when the
|
| 92 |
+
torchvision binary extension can't load against our CPU torch
|
| 93 |
+
wheel. Treat that as 'unavailable' too β the local inference path
|
| 94 |
+
is dead-on-arrival there."""
|
| 95 |
+
missing: list[str] = []
|
| 96 |
+
for name in ("terratorch", "peft", "safetensors", "huggingface_hub",
|
| 97 |
+
"torch", "yaml"):
|
| 98 |
+
try:
|
| 99 |
+
__import__(name)
|
| 100 |
+
except ImportError:
|
| 101 |
+
missing.append(name)
|
| 102 |
+
except Exception as e:
|
| 103 |
+
# torchvision::nms RuntimeError, libcuda load failure, etc.
|
| 104 |
+
log.warning("terramind_nyc: %s import raised %s; treating as "
|
| 105 |
+
"unavailable", name, type(e).__name__)
|
| 106 |
+
missing.append(f"{name} ({type(e).__name__})")
|
| 107 |
+
if missing:
|
| 108 |
+
return False, ", ".join(missing)
|
| 109 |
+
return True, None
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
_DEPS_OK, _DEPS_MISSING = _has_required_deps()
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def _ensure_adapter(adapter_name: str):
|
| 116 |
+
"""Build the terratorch SemanticSegmentationTask, inject the LoRA
|
| 117 |
+
scaffold, load the published Ξ + decoder weights, return the task.
|
| 118 |
+
|
| 119 |
+
Per-task tasks share the TerraMind base inside terratorch's model
|
| 120 |
+
factory β calling SemanticSegmentationTask twice loads the base
|
| 121 |
+
twice in fp32 (~3.3 GB resident on CPU). For a two-task family this
|
| 122 |
+
is acceptable; we don't need the cross-task weight sharing the
|
| 123 |
+
experiments/18 ensemble does. If memory becomes a problem, swap
|
| 124 |
+
this for a single-task / hot-swap-adapter implementation.
|
| 125 |
+
"""
|
| 126 |
+
if adapter_name not in ADAPTER_SPECS:
|
| 127 |
+
raise KeyError(f"unknown adapter {adapter_name!r}; "
|
| 128 |
+
f"expected one of {list(ADAPTER_SPECS)}")
|
| 129 |
+
if adapter_name in _ADAPTERS:
|
| 130 |
+
return _ADAPTERS[adapter_name]
|
| 131 |
+
|
| 132 |
+
with _INIT_LOCK:
|
| 133 |
+
if adapter_name in _ADAPTERS:
|
| 134 |
+
return _ADAPTERS[adapter_name]
|
| 135 |
+
|
| 136 |
+
spec = ADAPTER_SPECS[adapter_name]
|
| 137 |
+
log.info("terramind_nyc: building task for %s", adapter_name)
|
| 138 |
+
|
| 139 |
+
from huggingface_hub import snapshot_download
|
| 140 |
+
from peft import LoraConfig, inject_adapter_in_model
|
| 141 |
+
from safetensors.torch import load_file
|
| 142 |
+
from terratorch.tasks import SemanticSegmentationTask
|
| 143 |
+
|
| 144 |
+
# 1. Pull the requested adapter subtree from the HF repo.
|
| 145 |
+
adapter_root = snapshot_download(
|
| 146 |
+
ADAPTERS_REPO,
|
| 147 |
+
allow_patterns=[f"{spec['subdir']}/*"],
|
| 148 |
+
)
|
| 149 |
+
|
| 150 |
+
# 2. Build the standard terratorch task with the same model_args
|
| 151 |
+
# the published HF_README quick-start uses.
|
| 152 |
+
task = SemanticSegmentationTask(
|
| 153 |
+
model_factory="EncoderDecoderFactory",
|
| 154 |
+
model_args=dict(
|
| 155 |
+
backbone="terramind_v1_base",
|
| 156 |
+
backbone_pretrained=True,
|
| 157 |
+
backbone_modalities=["S2L2A", "S1RTC", "DEM"],
|
| 158 |
+
backbone_use_temporal=True,
|
| 159 |
+
backbone_temporal_pooling="concat",
|
| 160 |
+
backbone_temporal_n_timestamps=4,
|
| 161 |
+
necks=[
|
| 162 |
+
{"name": "SelectIndices", "indices": [2, 5, 8, 11]},
|
| 163 |
+
{"name": "ReshapeTokensToImage", "remove_cls_token": False},
|
| 164 |
+
{"name": "LearnedInterpolateToPyramidal"},
|
| 165 |
+
],
|
| 166 |
+
decoder="UNetDecoder",
|
| 167 |
+
decoder_channels=[512, 256, 128, 64],
|
| 168 |
+
head_dropout=0.1,
|
| 169 |
+
num_classes=spec["num_classes"],
|
| 170 |
+
),
|
| 171 |
+
loss="ce", lr=1e-4, freeze_backbone=False, freeze_decoder=False,
|
| 172 |
+
)
|
| 173 |
+
|
| 174 |
+
# 3. Inject the LoRA scaffold the adapter weights were trained
|
| 175 |
+
# against. Same hyperparameters every adapter in this family
|
| 176 |
+
# used (see experiments/18 adapters/_template/config.yaml).
|
| 177 |
+
inject_adapter_in_model(LoraConfig(
|
| 178 |
+
r=16, lora_alpha=32, lora_dropout=0.05,
|
| 179 |
+
target_modules=["attn.qkv", "attn.proj"], bias="none",
|
| 180 |
+
), task.model.encoder)
|
| 181 |
+
|
| 182 |
+
# 4. Restore Ξ matrices (encoder LoRA) and the decoder/neck/head
|
| 183 |
+
# weights from the safetensors bundle. The encoder.* prefix
|
| 184 |
+
# is stripped because the encoder state-dict is rooted at
|
| 185 |
+
# the encoder module, not the task.
|
| 186 |
+
adapter_dir = f"{adapter_root}/{spec['subdir']}"
|
| 187 |
+
lora_state = load_file(f"{adapter_dir}/adapter_model.safetensors")
|
| 188 |
+
head_state = load_file(f"{adapter_dir}/decoder_head.safetensors")
|
| 189 |
+
encoder_state = {
|
| 190 |
+
k.removeprefix("encoder."): v
|
| 191 |
+
for k, v in lora_state.items() if k.startswith("encoder.")
|
| 192 |
+
}
|
| 193 |
+
task.model.encoder.load_state_dict(encoder_state, strict=False)
|
| 194 |
+
for sub in ("decoder", "neck", "head", "aux_heads"):
|
| 195 |
+
sub_state = {
|
| 196 |
+
k[len(sub) + 1:]: v
|
| 197 |
+
for k, v in head_state.items() if k.startswith(sub + ".")
|
| 198 |
+
}
|
| 199 |
+
if sub_state and hasattr(task.model, sub):
|
| 200 |
+
getattr(task.model, sub).load_state_dict(sub_state,
|
| 201 |
+
strict=False)
|
| 202 |
+
|
| 203 |
+
# 5. Move to the configured device. CUDA only if the caller
|
| 204 |
+
# asked AND a CUDA device is actually available β silently
|
| 205 |
+
# fall back to CPU otherwise.
|
| 206 |
+
target_device = DEVICE
|
| 207 |
+
if target_device == "cuda":
|
| 208 |
+
import torch
|
| 209 |
+
if not torch.cuda.is_available():
|
| 210 |
+
log.warning("terramind_nyc: CUDA unavailable, falling back to CPU")
|
| 211 |
+
target_device = "cpu"
|
| 212 |
+
task = task.to(target_device).eval()
|
| 213 |
+
|
| 214 |
+
_ADAPTERS[adapter_name] = task
|
| 215 |
+
log.info("terramind_nyc: %s ready on %s", adapter_name, target_device)
|
| 216 |
+
return task
|
| 217 |
+
|
| 218 |
+
|
| 219 |
+
def _tiled_predict(task, modality_chips: dict, num_classes: int):
|
| 220 |
+
"""Run the task's encoder-decoder forward in 224Γ224 tiles, returning
|
| 221 |
+
a (1, num_classes, H, W) logits tensor stitched from the windows.
|
| 222 |
+
|
| 223 |
+
TerraMind's positional embeddings are tied to the 224Γ224 training
|
| 224 |
+
resolution. terratorch's tiled_inference helper slides a window
|
| 225 |
+
across the input modalities (it accepts a dict of per-modality
|
| 226 |
+
tensors as long as all modalities share HΓW), runs the model on
|
| 227 |
+
each crop, and averages overlapping logits. Without it, larger
|
| 228 |
+
chips return silent garbage; smaller chips error on the encoder
|
| 229 |
+
ViT.
|
| 230 |
+
"""
|
| 231 |
+
import torch
|
| 232 |
+
from terratorch.tasks.tiled_inference import tiled_inference
|
| 233 |
+
|
| 234 |
+
# tiled_inference invokes `model_forward(patch)` per tile. The task
|
| 235 |
+
# model returns a ModelOutput-like with .output OR a plain tensor;
|
| 236 |
+
# coerce to tensor either way.
|
| 237 |
+
def _forward(x, **_extra):
|
| 238 |
+
out = task.model(x)
|
| 239 |
+
return out.output if hasattr(out, "output") else out
|
| 240 |
+
|
| 241 |
+
with torch.no_grad():
|
| 242 |
+
logits = tiled_inference(
|
| 243 |
+
_forward,
|
| 244 |
+
modality_chips,
|
| 245 |
+
out_channels=num_classes,
|
| 246 |
+
h_crop=TILE_SIZE,
|
| 247 |
+
w_crop=TILE_SIZE,
|
| 248 |
+
h_stride=TILE_STRIDE,
|
| 249 |
+
w_stride=TILE_STRIDE,
|
| 250 |
+
average_patches=True,
|
| 251 |
+
blend_overlaps=True,
|
| 252 |
+
padding="reflect",
|
| 253 |
+
)
|
| 254 |
+
return logits
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
def _summarize_lulc(pred, class_labels: list[str]) -> dict[str, Any]:
|
| 258 |
+
"""Per-class pixel fraction + dominant class from an integer mask."""
|
| 259 |
+
import numpy as np
|
| 260 |
+
pred_np = pred.detach().cpu().numpy() if hasattr(pred, "detach") else np.asarray(pred)
|
| 261 |
+
flat = pred_np.reshape(-1)
|
| 262 |
+
n = max(int(flat.size), 1)
|
| 263 |
+
fractions: dict[str, float] = {}
|
| 264 |
+
for idx, label in enumerate(class_labels):
|
| 265 |
+
pct = 100.0 * float((flat == idx).sum()) / n
|
| 266 |
+
if pct > 0:
|
| 267 |
+
fractions[label] = round(pct, 2)
|
| 268 |
+
dominant_idx = int(max(range(len(class_labels)),
|
| 269 |
+
key=lambda i: int((flat == i).sum())))
|
| 270 |
+
return {
|
| 271 |
+
"ok": True,
|
| 272 |
+
"n_pixels": int(flat.size),
|
| 273 |
+
"shape": list(pred_np.shape),
|
| 274 |
+
"class_fractions": fractions,
|
| 275 |
+
"dominant_class": class_labels[dominant_idx],
|
| 276 |
+
"dominant_pct": fractions.get(class_labels[dominant_idx], 0.0),
|
| 277 |
+
}
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
def _summarize_buildings(pred, class_labels: list[str]) -> dict[str, Any]:
|
| 281 |
+
"""Building-pixel coverage + simple connected-component count."""
|
| 282 |
+
import numpy as np
|
| 283 |
+
pred_np = pred.detach().cpu().numpy() if hasattr(pred, "detach") else np.asarray(pred)
|
| 284 |
+
mask = (pred_np == 1).astype("uint8")
|
| 285 |
+
n_total = max(int(mask.size), 1)
|
| 286 |
+
pct_built = 100.0 * float(mask.sum()) / n_total
|
| 287 |
+
# Connected-component count is a cheap signal of "how many distinct
|
| 288 |
+
# buildings does this chip cover" β useful for the briefing without
|
| 289 |
+
# paying for full polygonisation.
|
| 290 |
+
n_components: int | None = None
|
| 291 |
+
try:
|
| 292 |
+
from scipy.ndimage import label
|
| 293 |
+
_, n_components = label(mask)
|
| 294 |
+
except Exception: # scipy is optional in some HF Spaces build cones
|
| 295 |
+
log.debug("terramind_nyc: scipy.ndimage unavailable; "
|
| 296 |
+
"skipping component count")
|
| 297 |
+
return {
|
| 298 |
+
"ok": True,
|
| 299 |
+
"n_pixels": int(mask.size),
|
| 300 |
+
"shape": list(mask.shape),
|
| 301 |
+
"pct_buildings": round(pct_built, 2),
|
| 302 |
+
"n_building_components": n_components,
|
| 303 |
+
"class_labels": class_labels,
|
| 304 |
+
}
|
| 305 |
+
|
| 306 |
+
|
| 307 |
+
def _try_remote(adapter_name: str, modality_chips: dict) -> dict | None:
|
| 308 |
+
"""POST to the riprap-models inference service if configured.
|
| 309 |
+
|
| 310 |
+
Returns:
|
| 311 |
+
- successful result dict on a 200/ok=True remote response
|
| 312 |
+
- {"ok": False, "skipped": "<reason>"} when remote was attempted
|
| 313 |
+
but failed (RemoteUnreachable, ok=False, or other error). The
|
| 314 |
+
caller MUST NOT fall through to local terratorch in this case
|
| 315 |
+
β local has been broken on the CPU-tier UI Spaces since the
|
| 316 |
+
torchvision binary mismatch landed, and we'd rather show a
|
| 317 |
+
clean "remote unreachable" reason than a noisy crash.
|
| 318 |
+
- None ONLY when remote isn't configured at all (caller may
|
| 319 |
+
legitimately try local then)."""
|
| 320 |
+
try:
|
| 321 |
+
from app import inference as _inf
|
| 322 |
+
if not _inf.remote_enabled():
|
| 323 |
+
return None
|
| 324 |
+
s2 = modality_chips.get("S2L2A")
|
| 325 |
+
s1 = modality_chips.get("S1RTC")
|
| 326 |
+
dem = modality_chips.get("DEM")
|
| 327 |
+
# The router serializes torch tensors to base64 numpy float32 β
|
| 328 |
+
# the chip cache hands us [B, C, T, H, W]; keep that shape, the
|
| 329 |
+
# service rebuilds the temporal stack on its end.
|
| 330 |
+
result = _inf.terramind(adapter_name, s2, s1, dem)
|
| 331 |
+
if not result.get("ok"):
|
| 332 |
+
err = result.get("error") or result.get("err") or "unknown"
|
| 333 |
+
return {"ok": False,
|
| 334 |
+
"skipped": f"remote terramind/{adapter_name} non-ok: {err}"}
|
| 335 |
+
result.setdefault("adapter", adapter_name)
|
| 336 |
+
result.setdefault("repo", ADAPTERS_REPO)
|
| 337 |
+
result["compute"] = f"remote Β· {result.get('device', 'gpu')}"
|
| 338 |
+
# Polygonize the prediction raster onto the chip's bounds so
|
| 339 |
+
# the map can paint the LULC / buildings overlay. Bounds come
|
| 340 |
+
# via the modality_chips dict β the eo_chip layer threads them
|
| 341 |
+
# through. Best-effort; never raises into the FSM.
|
| 342 |
+
bounds = modality_chips.get("bounds_4326") if modality_chips else None
|
| 343 |
+
pred_b64 = result.get("pred_b64")
|
| 344 |
+
pred_shape = result.get("pred_shape")
|
| 345 |
+
class_labels = result.get("class_labels")
|
| 346 |
+
if bounds and pred_b64 and pred_shape:
|
| 347 |
+
try:
|
| 348 |
+
from app.context._polygonize import (
|
| 349 |
+
polygonize_binary_mask, polygonize_class_raster,
|
| 350 |
+
)
|
| 351 |
+
if adapter_name == "buildings":
|
| 352 |
+
polys = polygonize_binary_mask(
|
| 353 |
+
pred_b64, pred_shape, tuple(bounds),
|
| 354 |
+
label="building", fill_color="#D62728",
|
| 355 |
+
simplify_tolerance=2e-5,
|
| 356 |
+
)
|
| 357 |
+
else:
|
| 358 |
+
polys = polygonize_class_raster(
|
| 359 |
+
pred_b64, pred_shape, class_labels, tuple(bounds),
|
| 360 |
+
simplify_tolerance=2e-5,
|
| 361 |
+
)
|
| 362 |
+
result["polygons_geojson"] = polys
|
| 363 |
+
except Exception:
|
| 364 |
+
log.exception("terramind/%s: polygonize failed", adapter_name)
|
| 365 |
+
result["polygons_geojson"] = None
|
| 366 |
+
return result
|
| 367 |
+
except _inf.RemoteUnreachable as e:
|
| 368 |
+
log.info("terramind/%s: remote unreachable (%s)", adapter_name, e)
|
| 369 |
+
return {"ok": False,
|
| 370 |
+
"skipped": f"remote terramind/{adapter_name} unreachable: {e}"}
|
| 371 |
+
except Exception as e:
|
| 372 |
+
log.exception("terramind/%s: remote call failed", adapter_name)
|
| 373 |
+
return {"ok": False,
|
| 374 |
+
"skipped": f"remote terramind/{adapter_name} error: "
|
| 375 |
+
f"{type(e).__name__}: {e}"}
|
| 376 |
+
|
| 377 |
+
|
| 378 |
+
def _run(adapter_name: str, modality_chips: dict, summarizer):
|
| 379 |
+
"""Common boilerplate: gate, time, [remote attempt], load, tiled
|
| 380 |
+
predict, summarize."""
|
| 381 |
+
if not ENABLE:
|
| 382 |
+
return {"ok": False,
|
| 383 |
+
"skipped": "RIPRAP_TERRAMIND_NYC_ENABLE=0"}
|
| 384 |
+
|
| 385 |
+
# v0.4.5 β try remote first. The remote service has its own deps,
|
| 386 |
+
# so this path works even when local _DEPS_OK is False (the most
|
| 387 |
+
# common HF Spaces case until terratorch + peft are baked in).
|
| 388 |
+
remote = _try_remote(adapter_name, modality_chips or {})
|
| 389 |
+
if remote is not None:
|
| 390 |
+
return remote
|
| 391 |
+
|
| 392 |
+
if not _DEPS_OK:
|
| 393 |
+
return {"ok": False,
|
| 394 |
+
"skipped": f"deps unavailable on this deployment: "
|
| 395 |
+
f"{_DEPS_MISSING}"}
|
| 396 |
+
if not modality_chips:
|
| 397 |
+
return {"ok": False, "err": "no modality chips supplied"}
|
| 398 |
+
t0 = time.time()
|
| 399 |
+
try:
|
| 400 |
+
task = _ensure_adapter(adapter_name)
|
| 401 |
+
spec = ADAPTER_SPECS[adapter_name]
|
| 402 |
+
# Strip out bounds_4326 (auxiliary metadata, not a tensor) before
|
| 403 |
+
# handing the dict to terratorch's tiled_inference, which iterates
|
| 404 |
+
# all values as modalities.
|
| 405 |
+
tensors_only = {k: v for k, v in modality_chips.items()
|
| 406 |
+
if k != "bounds_4326"}
|
| 407 |
+
logits = _tiled_predict(task, tensors_only, spec["num_classes"])
|
| 408 |
+
# logits: (B, C, H, W). Argmax to per-pixel class id.
|
| 409 |
+
pred = logits.argmax(dim=1).squeeze(0)
|
| 410 |
+
result = summarizer(pred, spec["class_labels"])
|
| 411 |
+
result["elapsed_s"] = round(time.time() - t0, 2)
|
| 412 |
+
result["adapter"] = adapter_name
|
| 413 |
+
result["repo"] = ADAPTERS_REPO
|
| 414 |
+
result["compute"] = "local"
|
| 415 |
+
return result
|
| 416 |
+
except Exception as e:
|
| 417 |
+
msg = str(e)
|
| 418 |
+
# Translate torchvision binary-extension failures into a clean
|
| 419 |
+
# skip. terratorch + torchvision both ride a transitive
|
| 420 |
+
# dep cone on the HF Space (sentence-transformers pulls torch
|
| 421 |
+
# CPU; torchvision's C extension can't load against that wheel),
|
| 422 |
+
# so a local _ensure_adapter() raises RuntimeError with this
|
| 423 |
+
# signature when remote is also unreachable. Clean skip is the
|
| 424 |
+
# honest demo outcome β same as terramind_synthesis.
|
| 425 |
+
if "torchvision::nms" in msg or "torchvision_C" in msg:
|
| 426 |
+
log.warning("terramind_nyc/%s: torchvision binary unavailable; "
|
| 427 |
+
"remote unreachable too; clean skip", adapter_name)
|
| 428 |
+
return {"ok": False,
|
| 429 |
+
"skipped": "remote inference unreachable + local "
|
| 430 |
+
"torchvision binary unavailable on this "
|
| 431 |
+
"deployment",
|
| 432 |
+
"elapsed_s": round(time.time() - t0, 2)}
|
| 433 |
+
log.exception("terramind_nyc.%s failed", adapter_name)
|
| 434 |
+
return {"ok": False, "err": f"{type(e).__name__}: {e}",
|
| 435 |
+
"elapsed_s": round(time.time() - t0, 2)}
|
| 436 |
+
|
| 437 |
+
|
| 438 |
+
def lulc(s2l2a, s1rtc=None, dem=None,
|
| 439 |
+
bounds_4326: tuple[float, float, float, float] | None = None,
|
| 440 |
+
) -> dict[str, Any]:
|
| 441 |
+
"""5-class NYC macro land-cover.
|
| 442 |
+
|
| 443 |
+
Inputs are torch tensors. The temporal models we trained expect
|
| 444 |
+
[C, T, H, W] (preferred) or [C, H, W] (will be expanded to T=1).
|
| 445 |
+
Pass S1 and DEM if you have them β the published adapter was
|
| 446 |
+
trained on the full triplet and accuracy degrades when modalities
|
| 447 |
+
are dropped.
|
| 448 |
+
|
| 449 |
+
`bounds_4326` is `(minlon, minlat, maxlon, maxlat)` of the chip
|
| 450 |
+
in WGS84; when provided, the LULC raster is polygonised onto the
|
| 451 |
+
chip's geographic extent so the map can render an overlay.
|
| 452 |
+
"""
|
| 453 |
+
chips = {"S2L2A": s2l2a}
|
| 454 |
+
if bounds_4326 is not None:
|
| 455 |
+
chips["bounds_4326"] = bounds_4326
|
| 456 |
+
if s1rtc is not None:
|
| 457 |
+
chips["S1RTC"] = s1rtc
|
| 458 |
+
if dem is not None:
|
| 459 |
+
chips["DEM"] = dem
|
| 460 |
+
return _run("lulc", chips, _summarize_lulc)
|
| 461 |
+
|
| 462 |
+
|
| 463 |
+
def buildings(s2l2a, s1rtc=None, dem=None,
|
| 464 |
+
bounds_4326: tuple[float, float, float, float] | None = None,
|
| 465 |
+
) -> dict[str, Any]:
|
| 466 |
+
"""Binary NYC building-footprint mask. Same input contract as lulc()."""
|
| 467 |
+
chips = {"S2L2A": s2l2a}
|
| 468 |
+
if bounds_4326 is not None:
|
| 469 |
+
chips["bounds_4326"] = bounds_4326
|
| 470 |
+
if s1rtc is not None:
|
| 471 |
+
chips["S1RTC"] = s1rtc
|
| 472 |
+
if dem is not None:
|
| 473 |
+
chips["DEM"] = dem
|
| 474 |
+
return _run("buildings", chips, _summarize_buildings)
|
| 475 |
+
|
| 476 |
+
|
| 477 |
+
def warm():
|
| 478 |
+
"""Optional pre-load β amortizes the first-query model build cost."""
|
| 479 |
+
if not ENABLE or not _DEPS_OK:
|
| 480 |
+
return
|
| 481 |
+
try:
|
| 482 |
+
for name in ADAPTER_SPECS:
|
| 483 |
+
_ensure_adapter(name)
|
| 484 |
+
except Exception:
|
| 485 |
+
log.exception("terramind_nyc: warm() failed; specialists will no-op")
|
app/context/terramind_synthesis.py
ADDED
|
@@ -0,0 +1,468 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""TerraMind v1 base as a real-time FSM node β DEM β ESRI LULC.
|
| 2 |
+
|
| 3 |
+
Per user query: take the geocoded (lat, lon), pull a DEM patch from
|
| 4 |
+
Riprap's existing NYC-wide LiDAR raster (already used by the microtopo
|
| 5 |
+
specialist β no STAC dependency), run TerraMind to generate a
|
| 6 |
+
plausible categorical land-cover map from the terrain context, and
|
| 7 |
+
emit class fractions the reconciler can cite as a synthetic-prior
|
| 8 |
+
context layer alongside the empirical and modeled flood evidence.
|
| 9 |
+
|
| 10 |
+
Why DEM β LULC (and not DEM β S2L2A as initially prototyped):
|
| 11 |
+
- LULC is *categorical* and *interpretable*. The output is one of
|
| 12 |
+
10 ESRI Land Cover classes per pixel; class fractions like "78%
|
| 13 |
+
Built Area" go straight into the briefing as cite-able claims.
|
| 14 |
+
- S2L2A is 12-channel reflectance β uninterpretable downstream
|
| 15 |
+
without a separate segmentation head.
|
| 16 |
+
- LULC is *comparable to ground truth*: NYC PLUTO land-use class
|
| 17 |
+
is already in the data layer; future calibration possible.
|
| 18 |
+
|
| 19 |
+
Class label mapping is *tentative* against ESRI 2020-2022 schema
|
| 20 |
+
(which TerraMesh's LULC tokenizer was trained on). The doc body
|
| 21 |
+
discloses the mapping as tentative and the reconciler is instructed
|
| 22 |
+
to use hedged framing ("the synthetic land-cover prior identifies β¦
|
| 23 |
+
likely class β¦") rather than asserting hard labels.
|
| 24 |
+
|
| 25 |
+
Why this shape:
|
| 26 |
+
- **No STAC dependency.** Microsoft Planetary Computer search has
|
| 27 |
+
been intermittent during this hackathon; the DEM raster is local
|
| 28 |
+
and always available.
|
| 29 |
+
- **Real-time.** < 0.3 s synthesis + < 0.5 s DEM patch read on M3
|
| 30 |
+
CPU once warm.
|
| 31 |
+
- **Honesty discipline.** Synthetic-prior tier, fourth epistemic
|
| 32 |
+
class alongside empirical / modeled / proxy.
|
| 33 |
+
|
| 34 |
+
License: Apache-2.0 β `ibm-esa-geospatial/TerraMind-1.0-base`.
|
| 35 |
+
"""
|
| 36 |
+
|
| 37 |
+
from __future__ import annotations
|
| 38 |
+
|
| 39 |
+
import logging
|
| 40 |
+
import os
|
| 41 |
+
import random
|
| 42 |
+
import threading
|
| 43 |
+
import time
|
| 44 |
+
from typing import Any
|
| 45 |
+
|
| 46 |
+
log = logging.getLogger("riprap.terramind")
|
| 47 |
+
|
| 48 |
+
ENABLE = os.environ.get("RIPRAP_TERRAMIND_ENABLE", "1").lower() in ("1", "true", "yes")
|
| 49 |
+
DEFAULT_STEPS = int(os.environ.get("RIPRAP_TERRAMIND_STEPS", "10"))
|
| 50 |
+
DEFAULT_SEED = int(os.environ.get("RIPRAP_TERRAMIND_SEED", "42"))
|
| 51 |
+
CHIP_PX = int(os.environ.get("RIPRAP_TERRAMIND_CHIP_PX", "224"))
|
| 52 |
+
CHIP_M = CHIP_PX * 30 # NYC DEM is at 30 m -> 6.72 km square
|
| 53 |
+
HALF_M = CHIP_M / 2
|
| 54 |
+
|
| 55 |
+
_MODEL = None
|
| 56 |
+
_INIT_LOCK = threading.Lock()
|
| 57 |
+
|
| 58 |
+
# Tentative ESRI 2020-2022 Land Cover class mapping for TerraMind v1's
|
| 59 |
+
# LULC tokenizer output (10 channels, argmax over channel axis -> class
|
| 60 |
+
# index 0-9). The README/docs don't expose the exact mapping and the
|
| 61 |
+
# tokenizer source confirms only "ESRI LULC" without a label table, so
|
| 62 |
+
# the names below are best-effort. The doc body discloses tentativeness.
|
| 63 |
+
LULC_CLASSES = [
|
| 64 |
+
"water", # 0
|
| 65 |
+
"trees", # 1
|
| 66 |
+
"grass", # 2
|
| 67 |
+
"flooded_vegetation", # 3
|
| 68 |
+
"crops", # 4
|
| 69 |
+
"scrub_shrub", # 5
|
| 70 |
+
"built_area", # 6
|
| 71 |
+
"bare_ground", # 7
|
| 72 |
+
"snow_ice", # 8
|
| 73 |
+
"clouds_or_no_data", # 9
|
| 74 |
+
]
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def _has_required_deps() -> tuple[bool, str | None]:
|
| 78 |
+
"""Probe deps. terramind_synthesis runs only locally (no remote path
|
| 79 |
+
in app/inference.py for DEM-driven synthesis), so it always needs
|
| 80 |
+
terratorch. On the HF Space terratorch isn't installed, so this
|
| 81 |
+
specialist returns a clean `skipped: deps unavailable` outcome.
|
| 82 |
+
|
| 83 |
+
Distinguishes a *truly missing* package (ModuleNotFoundError) from
|
| 84 |
+
a *transient race* (other ImportError β typically sklearn's
|
| 85 |
+
"partially initialized module" from concurrent imports)."""
|
| 86 |
+
missing = []
|
| 87 |
+
for name in ("terratorch", "rasterio"):
|
| 88 |
+
try:
|
| 89 |
+
__import__(name)
|
| 90 |
+
except ModuleNotFoundError:
|
| 91 |
+
missing.append(name)
|
| 92 |
+
except ImportError:
|
| 93 |
+
log.debug("terramind: import race on %s, will retry on demand", name)
|
| 94 |
+
except Exception as e:
|
| 95 |
+
# torchvision::nms RuntimeError on HF Space β local inference
|
| 96 |
+
# is unavailable; treat as missing so fetch() returns a clean
|
| 97 |
+
# skip rather than crashing in _ensure_model.
|
| 98 |
+
log.warning("terramind: %s import raised %s; treating as "
|
| 99 |
+
"unavailable", name, type(e).__name__)
|
| 100 |
+
missing.append(f"{name} ({type(e).__name__})")
|
| 101 |
+
return (not missing, ", ".join(missing) if missing else None)
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
_DEPS_OK, _DEPS_MISSING = _has_required_deps()
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def _ensure_model():
|
| 108 |
+
"""Lazy load with a lock so the parallel-block worker can't double-init."""
|
| 109 |
+
global _MODEL
|
| 110 |
+
if _MODEL is not None:
|
| 111 |
+
return _MODEL
|
| 112 |
+
with _INIT_LOCK:
|
| 113 |
+
if _MODEL is not None:
|
| 114 |
+
return _MODEL
|
| 115 |
+
# Heavy import deferred to first call so module import stays cheap
|
| 116 |
+
# and HF Spaces (no terratorch) doesn't pay it at all.
|
| 117 |
+
import terratorch.models.backbones.terramind.model.terramind_register # noqa
|
| 118 |
+
from terratorch.registry import FULL_MODEL_REGISTRY
|
| 119 |
+
log.info("terramind: loading v1 base generate (DEM -> LULC)")
|
| 120 |
+
m = FULL_MODEL_REGISTRY.build(
|
| 121 |
+
"terratorch_terramind_v1_base_generate",
|
| 122 |
+
modalities=["DEM"],
|
| 123 |
+
output_modalities=["LULC"],
|
| 124 |
+
pretrained=True,
|
| 125 |
+
timesteps=DEFAULT_STEPS,
|
| 126 |
+
)
|
| 127 |
+
m.eval()
|
| 128 |
+
_MODEL = m
|
| 129 |
+
log.info("terramind: model ready")
|
| 130 |
+
return _MODEL
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def warm():
|
| 134 |
+
"""Call at app boot to amortize the ~6 s checkpoint load + first-call
|
| 135 |
+
JIT. No-op when deps are absent."""
|
| 136 |
+
if ENABLE and _DEPS_OK:
|
| 137 |
+
try:
|
| 138 |
+
_ensure_model()
|
| 139 |
+
except Exception:
|
| 140 |
+
log.exception("terramind: warm() failed; specialist will no-op")
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def _read_dem_patch(lat: float, lon: float):
|
| 144 |
+
"""Read a CHIP_PXΓCHIP_PX DEM patch centered on (lat, lon) from the
|
| 145 |
+
local NYC-wide LiDAR raster. Returns (array, bounds_4326) where
|
| 146 |
+
bounds_4326 is (minlon, minlat, maxlon, maxlat) so the synthesised
|
| 147 |
+
LULC can be georeferenced onto the same extent for map rendering.
|
| 148 |
+
Returns None if outside the raster's extent."""
|
| 149 |
+
from pathlib import Path
|
| 150 |
+
|
| 151 |
+
import numpy as np
|
| 152 |
+
import rasterio
|
| 153 |
+
from rasterio.windows import from_bounds
|
| 154 |
+
dem_path = (Path(__file__).resolve().parents[2]
|
| 155 |
+
/ "data" / "nyc_dem_30m.tif")
|
| 156 |
+
if not dem_path.exists():
|
| 157 |
+
return None
|
| 158 |
+
with rasterio.open(dem_path) as src:
|
| 159 |
+
# The DEM is in EPSG:4326 (geographic) in our cache β convert
|
| 160 |
+
# the chip extent in the same CRS by building a rough degree
|
| 161 |
+
# bbox from a meters-square half-side at NYC latitude.
|
| 162 |
+
# 1 degree lat β 111 km, 1 degree lon β 85 km at 40.7Β°N.
|
| 163 |
+
d_lat = (HALF_M / 111_000.0)
|
| 164 |
+
d_lon = (HALF_M / 85_000.0)
|
| 165 |
+
win = from_bounds(lon - d_lon, lat - d_lat,
|
| 166 |
+
lon + d_lon, lat + d_lat,
|
| 167 |
+
src.transform)
|
| 168 |
+
arr = src.read(1, window=win, boundless=True, fill_value=0).astype("float32")
|
| 169 |
+
if arr.size == 0 or arr.shape[0] < 8 or arr.shape[1] < 8:
|
| 170 |
+
return None
|
| 171 |
+
# Resize to CHIP_PX Γ CHIP_PX via torch interpolation. The exact
|
| 172 |
+
# pixel-perfect alignment doesn't matter for a synthetic prior; the
|
| 173 |
+
# model just needs a real terrain patch to condition on.
|
| 174 |
+
import torch
|
| 175 |
+
t = torch.from_numpy(arr).unsqueeze(0).unsqueeze(0)
|
| 176 |
+
t = torch.nn.functional.interpolate(t, size=(CHIP_PX, CHIP_PX),
|
| 177 |
+
mode="bilinear", align_corners=False)
|
| 178 |
+
out = t.squeeze(0).numpy() # (1, CHIP_PX, CHIP_PX)
|
| 179 |
+
# Replace NaN sentinel values with median elevation so the model
|
| 180 |
+
# doesn't see NaN tokens.
|
| 181 |
+
if np.isnan(out).any():
|
| 182 |
+
med = float(np.nanmedian(out))
|
| 183 |
+
out = np.nan_to_num(out, nan=med)
|
| 184 |
+
bounds_4326 = (lon - d_lon, lat - d_lat, lon + d_lon, lat + d_lat)
|
| 185 |
+
return out, bounds_4326
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
# Map class index -> visual color for the categorical fill on the
|
| 189 |
+
# MapLibre layer. Colors picked to be visually distinct from the
|
| 190 |
+
# existing red (Sandy) / blue (DEP) / cyan (Prithvi) / orange (Ida HWM).
|
| 191 |
+
LULC_FILL_COLORS = {
|
| 192 |
+
"water": "#0284c7", # not used (we keep water clear so
|
| 193 |
+
# the underlying basemap shows)
|
| 194 |
+
"trees": "#16a34a", # green
|
| 195 |
+
"grass": "#86efac", # pale green
|
| 196 |
+
"flooded_vegetation": "#a3e635", # lime
|
| 197 |
+
"crops": "#fde047", # yellow
|
| 198 |
+
"scrub_shrub": "#bef264",
|
| 199 |
+
"built_area": "#9ca3af", # neutral gray
|
| 200 |
+
"bare_ground": "#d6d3d1", # warm light gray
|
| 201 |
+
"snow_ice": "#f3f4f6",
|
| 202 |
+
"clouds_or_no_data": "#000000", # not used (kept transparent)
|
| 203 |
+
}
|
| 204 |
+
# Classes we don't render at all (transparent) β water is best left
|
| 205 |
+
# uncolored so the basemap shoreline reads through; clouds/no-data is
|
| 206 |
+
# semantically meaningless to fill.
|
| 207 |
+
LULC_HIDE_CLASSES = {"water", "clouds_or_no_data"}
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
def _polygonize_lulc(class_idx, bounds_4326: tuple) -> dict:
|
| 211 |
+
"""Vectorize the per-pixel argmax classification into one MultiPolygon
|
| 212 |
+
per class label, then dump as a single GeoJSON FeatureCollection in
|
| 213 |
+
EPSG:4326. Each feature carries `label` + `class_idx` properties so
|
| 214 |
+
the frontend can colour by category.
|
| 215 |
+
"""
|
| 216 |
+
import json
|
| 217 |
+
|
| 218 |
+
import geopandas as gpd
|
| 219 |
+
from rasterio.features import shapes
|
| 220 |
+
from rasterio.transform import from_bounds as transform_from_bounds
|
| 221 |
+
from shapely.geometry import shape
|
| 222 |
+
|
| 223 |
+
minlon, minlat, maxlon, maxlat = bounds_4326
|
| 224 |
+
h, w = class_idx.shape
|
| 225 |
+
transform = transform_from_bounds(minlon, minlat, maxlon, maxlat, w, h)
|
| 226 |
+
feats = []
|
| 227 |
+
for i, label in enumerate(LULC_CLASSES):
|
| 228 |
+
if label in LULC_HIDE_CLASSES:
|
| 229 |
+
continue
|
| 230 |
+
mask = (class_idx == i).astype("uint8")
|
| 231 |
+
if mask.sum() < 8: # skip tiny noise
|
| 232 |
+
continue
|
| 233 |
+
polys = []
|
| 234 |
+
for geom, value in shapes(mask, mask=mask.astype(bool),
|
| 235 |
+
transform=transform):
|
| 236 |
+
if value != 1:
|
| 237 |
+
continue
|
| 238 |
+
polys.append(shape(geom))
|
| 239 |
+
if not polys:
|
| 240 |
+
continue
|
| 241 |
+
# Dissolve via geopandas + simplify lightly. The chip is 30 m
|
| 242 |
+
# per pixel and we don't need pixel-edge fidelity at urban zoom.
|
| 243 |
+
gdf = gpd.GeoDataFrame({"geometry": polys}, crs="EPSG:4326")
|
| 244 |
+
gdf["geometry"] = gdf.geometry.simplify(1e-4, preserve_topology=True)
|
| 245 |
+
for geom in gdf.geometry:
|
| 246 |
+
feats.append({
|
| 247 |
+
"type": "Feature",
|
| 248 |
+
"geometry": json.loads(gpd.GeoSeries([geom],
|
| 249 |
+
crs="EPSG:4326").to_json())["features"][0]["geometry"],
|
| 250 |
+
"properties": {"label": label, "class_idx": i,
|
| 251 |
+
"fill_color": LULC_FILL_COLORS.get(label, "#9ca3af")},
|
| 252 |
+
})
|
| 253 |
+
return {"type": "FeatureCollection", "features": feats}
|
| 254 |
+
|
| 255 |
+
|
| 256 |
+
def fetch(lat: float, lon: float, timeout_s: float = 60.0) -> dict[str, Any]:
|
| 257 |
+
"""Run the specialist. Returns:
|
| 258 |
+
{ ok: bool,
|
| 259 |
+
skipped: str | None,
|
| 260 |
+
synthetic_modality: bool,
|
| 261 |
+
tim_chain: list[str],
|
| 262 |
+
diffusion_steps: int, diffusion_seed: int,
|
| 263 |
+
dem_mean_m: float,
|
| 264 |
+
class_fractions: dict[str, float], # tentative ESRI labels
|
| 265 |
+
dominant_class: str, # highest-fraction label
|
| 266 |
+
dominant_pct: float,
|
| 267 |
+
n_classes_observed: int,
|
| 268 |
+
chip_shape: list[int],
|
| 269 |
+
elapsed_s: float,
|
| 270 |
+
err: str | None }
|
| 271 |
+
|
| 272 |
+
Designed never to raise. Failures show up as ok=False with reason.
|
| 273 |
+
"""
|
| 274 |
+
if not ENABLE:
|
| 275 |
+
return {"ok": False, "skipped": "RIPRAP_TERRAMIND_ENABLE=0"}
|
| 276 |
+
t0 = time.time()
|
| 277 |
+
try:
|
| 278 |
+
import numpy as np
|
| 279 |
+
patch = _read_dem_patch(lat, lon)
|
| 280 |
+
if patch is None:
|
| 281 |
+
return {"ok": False, "skipped": "no DEM coverage at this point"}
|
| 282 |
+
dem, bounds_4326 = patch
|
| 283 |
+
dem_mean = float(dem.mean())
|
| 284 |
+
|
| 285 |
+
# v0.4.5+ β try the MI300X inference service first if configured.
|
| 286 |
+
# The droplet's /v1/terramind dispatch handles adapter='synthesis'
|
| 287 |
+
# via _terramind_synthesis_inference (DEM -> generative LULC). On
|
| 288 |
+
# the HF Space terratorch's torchvision binary doesn't load, so
|
| 289 |
+
# this is the only working path there.
|
| 290 |
+
try:
|
| 291 |
+
from app import inference as _inf
|
| 292 |
+
if _inf.remote_enabled():
|
| 293 |
+
# The terramind v1 base generative encoder embedding
|
| 294 |
+
# layer unpacks `B, C, H, W = x.shape` (verified against
|
| 295 |
+
# terratorch_terramind_v1_base_generate). DEM has C=1, so
|
| 296 |
+
# the on-the-wire shape is (1, 1, H, W) 4-D.
|
| 297 |
+
# `_read_dem_patch` returns a 3-D (1, H, W) array (it
|
| 298 |
+
# interpolates to CHIP_PXΓCHIP_PX through a 4-D
|
| 299 |
+
# torch.functional.interpolate then squeezes the batch),
|
| 300 |
+
# so we add only the batch dim β not two.
|
| 301 |
+
import numpy as _np_local
|
| 302 |
+
dem_arr = _np_local.asarray(dem, dtype="float32")
|
| 303 |
+
if dem_arr.ndim == 2: # (H, W)
|
| 304 |
+
dem_remote = dem_arr[None, None, :, :]
|
| 305 |
+
elif dem_arr.ndim == 3: # (1, H, W)
|
| 306 |
+
dem_remote = dem_arr[None, :, :, :]
|
| 307 |
+
elif dem_arr.ndim == 4: # already (1, 1, H, W)
|
| 308 |
+
dem_remote = dem_arr
|
| 309 |
+
else:
|
| 310 |
+
raise ValueError(
|
| 311 |
+
f"unexpected DEM shape {dem_arr.shape}; "
|
| 312 |
+
"expected 2/3/4-D")
|
| 313 |
+
remote = _inf.terramind("synthesis", None, None, dem_remote,
|
| 314 |
+
timeout=timeout_s)
|
| 315 |
+
if remote.get("ok"):
|
| 316 |
+
elapsed = round(time.time() - t0, 2)
|
| 317 |
+
# Polygonize the prediction raster for the map
|
| 318 |
+
# layer. The droplet returns the per-pixel argmax;
|
| 319 |
+
# we vectorize against the chip's bounds.
|
| 320 |
+
polys = None
|
| 321 |
+
pred_b64 = remote.get("pred_b64")
|
| 322 |
+
pred_shape = remote.get("pred_shape")
|
| 323 |
+
class_labels = (remote.get("class_labels")
|
| 324 |
+
or LULC_CLASSES)
|
| 325 |
+
if pred_b64 and pred_shape:
|
| 326 |
+
try:
|
| 327 |
+
from app.context._polygonize import (
|
| 328 |
+
polygonize_class_raster,
|
| 329 |
+
)
|
| 330 |
+
polys = polygonize_class_raster(
|
| 331 |
+
pred_b64, pred_shape, class_labels,
|
| 332 |
+
tuple(bounds_4326),
|
| 333 |
+
simplify_tolerance=2e-5,
|
| 334 |
+
)
|
| 335 |
+
except Exception:
|
| 336 |
+
log.exception("terramind/synthesis: "
|
| 337 |
+
"polygonize failed")
|
| 338 |
+
polys = None
|
| 339 |
+
out = {
|
| 340 |
+
"ok": True,
|
| 341 |
+
"synthetic_modality": True,
|
| 342 |
+
"tim_chain": ["DEM", "LULC_synthetic"],
|
| 343 |
+
"diffusion_steps": remote.get("diffusion_steps",
|
| 344 |
+
DEFAULT_STEPS),
|
| 345 |
+
"diffusion_seed": DEFAULT_SEED,
|
| 346 |
+
"dem_mean_m": round(dem_mean, 2),
|
| 347 |
+
"class_fractions": remote.get("class_fractions") or {},
|
| 348 |
+
"dominant_class": remote.get("dominant_class") or "unknown",
|
| 349 |
+
"dominant_pct": remote.get("dominant_pct") or 0.0,
|
| 350 |
+
"n_classes_observed": remote.get("n_classes_observed") or 0,
|
| 351 |
+
"chip_shape": remote.get("shape") or [],
|
| 352 |
+
"bounds_4326": list(bounds_4326),
|
| 353 |
+
"polygons_geojson": polys,
|
| 354 |
+
"label_schema": remote.get("label_schema") or "",
|
| 355 |
+
"compute": f"remote Β· {remote.get('device', 'gpu')}",
|
| 356 |
+
"elapsed_s": elapsed,
|
| 357 |
+
}
|
| 358 |
+
return out
|
| 359 |
+
# remote returned non-ok β surface that signal directly
|
| 360 |
+
return {"ok": False,
|
| 361 |
+
"skipped": f"remote terramind synthesis non-ok: "
|
| 362 |
+
f"{remote.get('error') or remote.get('detail') or 'unknown'}",
|
| 363 |
+
"elapsed_s": round(time.time() - t0, 2)}
|
| 364 |
+
except _inf.RemoteUnreachable as e:
|
| 365 |
+
log.info("terramind_synthesis: remote unreachable (%s); local fallback", e)
|
| 366 |
+
except Exception as e:
|
| 367 |
+
log.exception("terramind_synthesis: remote call failed")
|
| 368 |
+
return {"ok": False,
|
| 369 |
+
"skipped": f"remote terramind synthesis error: "
|
| 370 |
+
f"{type(e).__name__}: {e}",
|
| 371 |
+
"elapsed_s": round(time.time() - t0, 2)}
|
| 372 |
+
|
| 373 |
+
# Local fallback β original path; only available where terratorch
|
| 374 |
+
# imports without the torchvision::nms RuntimeError.
|
| 375 |
+
if not _DEPS_OK:
|
| 376 |
+
return {"ok": False, "skipped": f"deps unavailable: {_DEPS_MISSING}"}
|
| 377 |
+
import torch
|
| 378 |
+
random.seed(DEFAULT_SEED)
|
| 379 |
+
torch.manual_seed(DEFAULT_SEED)
|
| 380 |
+
|
| 381 |
+
model = _ensure_model()
|
| 382 |
+
# `dem` is 2-D (H, W) from `_read_dem_patch.src.read(1, ...)`. The
|
| 383 |
+
# terramind v1 base generative encoder wants (B=1, C=1, H, W) 4-D.
|
| 384 |
+
dem_t = torch.from_numpy(dem).unsqueeze(0).unsqueeze(0).float()
|
| 385 |
+
if time.time() - t0 > timeout_s:
|
| 386 |
+
return {"ok": False, "skipped": "terramind exceeded budget"}
|
| 387 |
+
|
| 388 |
+
with torch.no_grad():
|
| 389 |
+
out = model({"DEM": dem_t}, timesteps=DEFAULT_STEPS,
|
| 390 |
+
verbose=False)
|
| 391 |
+
lulc = out["LULC"]
|
| 392 |
+
if hasattr(lulc, "detach"):
|
| 393 |
+
lulc = lulc.detach().cpu().numpy()
|
| 394 |
+
if lulc.ndim == 4:
|
| 395 |
+
lulc = lulc[0] # (n_classes, H, W)
|
| 396 |
+
# Argmax over class channel -> per-pixel class index, then
|
| 397 |
+
# fraction by class. This is the cite-able structured output.
|
| 398 |
+
class_idx = lulc.argmax(axis=0) # (H, W)
|
| 399 |
+
unique, counts = np.unique(class_idx, return_counts=True)
|
| 400 |
+
total = float(class_idx.size)
|
| 401 |
+
fractions: dict[str, float] = {}
|
| 402 |
+
for u, c in zip(unique, counts, strict=False):
|
| 403 |
+
label = (LULC_CLASSES[int(u)] if 0 <= int(u) < len(LULC_CLASSES)
|
| 404 |
+
else f"class_{int(u)}")
|
| 405 |
+
fractions[label] = round(100.0 * c / total, 2)
|
| 406 |
+
# Sort dominant -> tail for deterministic doc body ordering.
|
| 407 |
+
ordered = dict(sorted(fractions.items(),
|
| 408 |
+
key=lambda kv: kv[1], reverse=True))
|
| 409 |
+
dominant_class = next(iter(ordered)) if ordered else "unknown"
|
| 410 |
+
dominant_pct = ordered.get(dominant_class, 0.0)
|
| 411 |
+
# Class indices map to TerraMesh's LULC tokenizer codebook; the
|
| 412 |
+
# exact label-to-index mapping isn't published. Surface a tentative
|
| 413 |
+
# name plus the raw index so a reader can see we're not asserting
|
| 414 |
+
# ground truth.
|
| 415 |
+
dominant_idx = next((i for i, lbl in enumerate(LULC_CLASSES)
|
| 416 |
+
if lbl == dominant_class), -1)
|
| 417 |
+
dominant_display = (
|
| 418 |
+
f"class_{dominant_idx} (tentative: {dominant_class})"
|
| 419 |
+
if dominant_idx >= 0 else dominant_class
|
| 420 |
+
)
|
| 421 |
+
|
| 422 |
+
# Polygonize the categorical raster for the map layer.
|
| 423 |
+
# Best-effort β failure here doesn't fail the specialist.
|
| 424 |
+
try:
|
| 425 |
+
polygons_geojson = _polygonize_lulc(class_idx, bounds_4326)
|
| 426 |
+
except Exception:
|
| 427 |
+
log.exception("terramind: polygonize failed; skipping map layer")
|
| 428 |
+
polygons_geojson = None
|
| 429 |
+
|
| 430 |
+
return {
|
| 431 |
+
"ok": True,
|
| 432 |
+
"synthetic_modality": True,
|
| 433 |
+
"tim_chain": ["DEM", "LULC_synthetic"],
|
| 434 |
+
"diffusion_steps": DEFAULT_STEPS,
|
| 435 |
+
"diffusion_seed": DEFAULT_SEED,
|
| 436 |
+
"dem_mean_m": round(dem_mean, 2),
|
| 437 |
+
"class_fractions": ordered,
|
| 438 |
+
"dominant_class": dominant_class,
|
| 439 |
+
"dominant_class_display": dominant_display,
|
| 440 |
+
"dominant_pct": dominant_pct,
|
| 441 |
+
"n_classes_observed": len(ordered),
|
| 442 |
+
"chip_shape": list(lulc.shape),
|
| 443 |
+
"bounds_4326": list(bounds_4326),
|
| 444 |
+
"polygons_geojson": polygons_geojson,
|
| 445 |
+
"label_schema": "ESRI 2020-2022 Land Cover (tentative β "
|
| 446 |
+
"TerraMind tokenizer source confirms ESRI but "
|
| 447 |
+
"not exact label-to-index mapping)",
|
| 448 |
+
"elapsed_s": round(time.time() - t0, 2),
|
| 449 |
+
}
|
| 450 |
+
except Exception as e:
|
| 451 |
+
msg = str(e)
|
| 452 |
+
# Translate the torchvision binary-extension failure into a clean
|
| 453 |
+
# skip. The HF Space ships torchvision via a transitive sentence-
|
| 454 |
+
# transformers dep, but its C extension can't load alongside our
|
| 455 |
+
# CPU torch wheel, so terratorch's NMS call raises RuntimeError.
|
| 456 |
+
# Surface this honestly β the local inference path is unavailable
|
| 457 |
+
# on this deployment, same outcome as a missing terratorch.
|
| 458 |
+
if "torchvision::nms" in msg or "torchvision_C" in msg:
|
| 459 |
+
log.warning("terramind: torchvision binary unavailable on this "
|
| 460 |
+
"deployment; skipping local inference")
|
| 461 |
+
return {"ok": False,
|
| 462 |
+
"skipped": "local inference unavailable on this "
|
| 463 |
+
"deployment (torchvision binary extension "
|
| 464 |
+
"not loadable); no remote synthesis path",
|
| 465 |
+
"elapsed_s": round(time.time() - t0, 2)}
|
| 466 |
+
log.exception("terramind: fetch failed")
|
| 467 |
+
return {"ok": False, "err": f"{type(e).__name__}: {e}",
|
| 468 |
+
"elapsed_s": round(time.time() - t0, 2)}
|
app/emissions.py
ADDED
|
@@ -0,0 +1,269 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Per-query emissions tracker for inference calls.
|
| 2 |
+
|
| 3 |
+
Records every LLM and ML-inference call made during a single query and
|
| 4 |
+
summarizes:
|
| 5 |
+
- wallclock duration per call
|
| 6 |
+
- prompt + completion tokens (LLM)
|
| 7 |
+
- energy in watt-hours, **measured from the L4 GPU when available**
|
| 8 |
+
(the inference proxy reports per-call `X-GPU-Power-W` /
|
| 9 |
+
`X-GPU-Energy-J` headers from a 100 ms-cadence NVML sampler).
|
| 10 |
+
Falls back to a duration Γ data-sheet-power estimate when the
|
| 11 |
+
proxy is unreachable / NVML init failed / call went to a backend
|
| 12 |
+
that doesn't surface power readings.
|
| 13 |
+
|
| 14 |
+
Each call record carries a `measured: bool` flag indicating which path
|
| 15 |
+
was used, so the UI can disclose. `summarize()` aggregates total Wh,
|
| 16 |
+
total tokens, by-kind and by-hardware splits β no cloud comparison.
|
| 17 |
+
|
| 18 |
+
Thread propagation
|
| 19 |
+
------------------
|
| 20 |
+
The tracker is held in a thread-local. The dispatch layer
|
| 21 |
+
(web/main.py) installs one per request; `app/fsm.py:iter_steps`
|
| 22 |
+
captures and re-installs it on the FSM runner thread (mirroring the
|
| 23 |
+
existing `_captured_token_cb` pattern). Worker threads spawned inside
|
| 24 |
+
specialists (prithvi_live, eo_chip_cache) inherit nothing β those calls
|
| 25 |
+
are silently dropped, which is acceptable: those specialists do <1 s of
|
| 26 |
+
inference each and are off the hot path for the energy story.
|
| 27 |
+
"""
|
| 28 |
+
from __future__ import annotations
|
| 29 |
+
|
| 30 |
+
import threading
|
| 31 |
+
from typing import Any
|
| 32 |
+
|
| 33 |
+
# (label, fallback_sustained_power_w, source). Used only when the
|
| 34 |
+
# proxy doesn't surface a real measurement (NVML disabled, backend
|
| 35 |
+
# unreachable, local-fallback path). The fallback figure is a
|
| 36 |
+
# conservative public-record estimate; the `measured: bool` flag on
|
| 37 |
+
# each call record indicates whether the row used the fallback.
|
| 38 |
+
HARDWARE: dict[str, tuple[str, float, str]] = {
|
| 39 |
+
"nvidia_l4": (
|
| 40 |
+
"NVIDIA L4",
|
| 41 |
+
60.0,
|
| 42 |
+
"NVIDIA L4 Tensor Core GPU data sheet (72 W TGP, Ada Lovelace, "
|
| 43 |
+
"24 GB); ~60 W sustained during transformer inference. The "
|
| 44 |
+
"active backend for the Riprap inference Space "
|
| 45 |
+
"(msradam/riprap-vllm). When the proxy is reachable and NVML "
|
| 46 |
+
"is initialized, real per-call power is read off the device "
|
| 47 |
+
"via nvmlDeviceGetPowerUsage and this fallback is unused.",
|
| 48 |
+
),
|
| 49 |
+
"amd_mi300x": (
|
| 50 |
+
"AMD MI300X",
|
| 51 |
+
600.0,
|
| 52 |
+
"AMD Instinct MI300X data sheet (750 W TDP); ~600 W sustained "
|
| 53 |
+
"during vLLM generation. Selected only when an operator deploys "
|
| 54 |
+
"against an MI300X droplet and sets RIPRAP_HARDWARE_LABEL=AMD "
|
| 55 |
+
"MI300X explicitly. The hackathon submission used to run on "
|
| 56 |
+
"this hardware; the droplet was decommissioned 2026-05-06.",
|
| 57 |
+
),
|
| 58 |
+
"nvidia_t4": (
|
| 59 |
+
"NVIDIA T4",
|
| 60 |
+
50.0,
|
| 61 |
+
"NVIDIA T4 data sheet (70 W max); ~50 W sustained during "
|
| 62 |
+
"transformer inference.",
|
| 63 |
+
),
|
| 64 |
+
"apple_m": (
|
| 65 |
+
"Apple M-series",
|
| 66 |
+
20.0,
|
| 67 |
+
"ml.energy / community measurements: ~20 W package power "
|
| 68 |
+
"during Granite 4.1 q4_K_M inference on Apple M3/M4 (the "
|
| 69 |
+
"local-dev path, no remote backend configured).",
|
| 70 |
+
),
|
| 71 |
+
"cpu_server": (
|
| 72 |
+
"x86 CPU",
|
| 73 |
+
30.0,
|
| 74 |
+
"Typical sustained x86 server-core load (~30 W) for CPU-only "
|
| 75 |
+
"inference fallbacks.",
|
| 76 |
+
),
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def _wh(power_w: float, duration_s: float) -> float:
|
| 81 |
+
return power_w * max(duration_s, 0.0) / 3600.0
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
class Tracker:
|
| 85 |
+
"""Append-only call ledger for one query. Thread-safe."""
|
| 86 |
+
|
| 87 |
+
def __init__(self) -> None:
|
| 88 |
+
self.calls: list[dict[str, Any]] = []
|
| 89 |
+
self._lock = threading.Lock()
|
| 90 |
+
|
| 91 |
+
def _record(self, *, base: dict[str, Any], hardware: str,
|
| 92 |
+
duration_s: float,
|
| 93 |
+
joules_real: float | None,
|
| 94 |
+
power_w_real: float | None) -> None:
|
| 95 |
+
"""Shared body of record_llm / record_ml.
|
| 96 |
+
|
| 97 |
+
When `joules_real` is provided (NVML-derived from the proxy),
|
| 98 |
+
we use it directly and stamp `measured=True`. Otherwise we
|
| 99 |
+
fall back to the data-sheet sustained-power estimate.
|
| 100 |
+
"""
|
| 101 |
+
hw_label, fallback_w, _src = HARDWARE.get(hardware,
|
| 102 |
+
HARDWARE["cpu_server"])
|
| 103 |
+
if joules_real is not None and joules_real >= 0:
|
| 104 |
+
joules = float(joules_real)
|
| 105 |
+
wh = joules / 3600.0
|
| 106 |
+
measured = True
|
| 107 |
+
avg_w = (joules / duration_s) if duration_s > 0 else (
|
| 108 |
+
power_w_real if power_w_real is not None else fallback_w)
|
| 109 |
+
else:
|
| 110 |
+
avg_w = fallback_w
|
| 111 |
+
wh = _wh(avg_w, duration_s)
|
| 112 |
+
joules = wh * 3600.0
|
| 113 |
+
measured = False
|
| 114 |
+
record = {
|
| 115 |
+
**base,
|
| 116 |
+
"hardware": hardware,
|
| 117 |
+
"hardware_label": hw_label,
|
| 118 |
+
"power_w": round(avg_w, 2),
|
| 119 |
+
"duration_s": round(duration_s, 3),
|
| 120 |
+
"measured": measured,
|
| 121 |
+
"wh": round(wh, 5),
|
| 122 |
+
"joules": round(joules, 3),
|
| 123 |
+
}
|
| 124 |
+
with self._lock:
|
| 125 |
+
self.calls.append(record)
|
| 126 |
+
|
| 127 |
+
def record_llm(self, *, model: str, backend: str, hardware: str,
|
| 128 |
+
prompt_tokens: int | None,
|
| 129 |
+
completion_tokens: int | None,
|
| 130 |
+
duration_s: float,
|
| 131 |
+
stream: bool = False,
|
| 132 |
+
joules_real: float | None = None,
|
| 133 |
+
power_w_real: float | None = None) -> None:
|
| 134 |
+
total = None
|
| 135 |
+
if prompt_tokens is not None or completion_tokens is not None:
|
| 136 |
+
total = (prompt_tokens or 0) + (completion_tokens or 0)
|
| 137 |
+
self._record(
|
| 138 |
+
base={
|
| 139 |
+
"kind": "llm",
|
| 140 |
+
"model": model,
|
| 141 |
+
"backend": backend,
|
| 142 |
+
"prompt_tokens": prompt_tokens,
|
| 143 |
+
"completion_tokens": completion_tokens,
|
| 144 |
+
"total_tokens": total,
|
| 145 |
+
"stream": stream,
|
| 146 |
+
},
|
| 147 |
+
hardware=hardware,
|
| 148 |
+
duration_s=duration_s,
|
| 149 |
+
joules_real=joules_real,
|
| 150 |
+
power_w_real=power_w_real,
|
| 151 |
+
)
|
| 152 |
+
|
| 153 |
+
def record_ml(self, *, endpoint: str, backend: str, hardware: str,
|
| 154 |
+
duration_s: float,
|
| 155 |
+
joules_real: float | None = None,
|
| 156 |
+
power_w_real: float | None = None) -> None:
|
| 157 |
+
self._record(
|
| 158 |
+
base={
|
| 159 |
+
"kind": "ml",
|
| 160 |
+
"endpoint": endpoint,
|
| 161 |
+
"backend": backend,
|
| 162 |
+
},
|
| 163 |
+
hardware=hardware,
|
| 164 |
+
duration_s=duration_s,
|
| 165 |
+
joules_real=joules_real,
|
| 166 |
+
power_w_real=power_w_real,
|
| 167 |
+
)
|
| 168 |
+
|
| 169 |
+
def summarize(self) -> dict[str, Any]:
|
| 170 |
+
with self._lock:
|
| 171 |
+
calls = list(self.calls)
|
| 172 |
+
total_wh = sum(c["wh"] for c in calls)
|
| 173 |
+
total_dur = sum(c["duration_s"] for c in calls)
|
| 174 |
+
n_measured = sum(1 for c in calls if c.get("measured"))
|
| 175 |
+
prompt = sum((c.get("prompt_tokens") or 0)
|
| 176 |
+
for c in calls if c["kind"] == "llm")
|
| 177 |
+
completion = sum((c.get("completion_tokens") or 0)
|
| 178 |
+
for c in calls if c["kind"] == "llm")
|
| 179 |
+
|
| 180 |
+
by_kind: dict[str, dict[str, Any]] = {}
|
| 181 |
+
for c in calls:
|
| 182 |
+
slot = by_kind.setdefault(c["kind"], {"wh": 0.0, "n": 0,
|
| 183 |
+
"duration_s": 0.0})
|
| 184 |
+
slot["wh"] += c["wh"]
|
| 185 |
+
slot["n"] += 1
|
| 186 |
+
slot["duration_s"] += c["duration_s"]
|
| 187 |
+
for slot in by_kind.values():
|
| 188 |
+
slot["wh"] = round(slot["wh"], 5)
|
| 189 |
+
slot["mwh"] = round(slot["wh"] * 1000, 2)
|
| 190 |
+
slot["duration_s"] = round(slot["duration_s"], 3)
|
| 191 |
+
|
| 192 |
+
by_hw: dict[str, dict[str, Any]] = {}
|
| 193 |
+
for c in calls:
|
| 194 |
+
slot = by_hw.setdefault(c["hardware"], {
|
| 195 |
+
"label": c["hardware_label"],
|
| 196 |
+
"wh": 0.0, "n": 0, "duration_s": 0.0,
|
| 197 |
+
})
|
| 198 |
+
slot["wh"] += c["wh"]
|
| 199 |
+
slot["n"] += 1
|
| 200 |
+
slot["duration_s"] += c["duration_s"]
|
| 201 |
+
for slot in by_hw.values():
|
| 202 |
+
slot["wh"] = round(slot["wh"], 5)
|
| 203 |
+
slot["mwh"] = round(slot["wh"] * 1000, 2)
|
| 204 |
+
slot["duration_s"] = round(slot["duration_s"], 3)
|
| 205 |
+
|
| 206 |
+
return {
|
| 207 |
+
"n_calls": len(calls),
|
| 208 |
+
"n_measured": n_measured,
|
| 209 |
+
"total_wh": round(total_wh, 5),
|
| 210 |
+
"total_mwh": round(total_wh * 1000, 2),
|
| 211 |
+
"total_joules": round(total_wh * 3600, 1),
|
| 212 |
+
"total_duration_s": round(total_dur, 3),
|
| 213 |
+
"tokens": {
|
| 214 |
+
"prompt": prompt or None,
|
| 215 |
+
"completion": completion or None,
|
| 216 |
+
"total": (prompt + completion) or None,
|
| 217 |
+
},
|
| 218 |
+
"by_kind": by_kind,
|
| 219 |
+
"by_hardware": by_hw,
|
| 220 |
+
"calls": calls,
|
| 221 |
+
"method": (
|
| 222 |
+
"Energy is read off the L4 GPU per call via "
|
| 223 |
+
"nvmlDeviceGetPowerUsage on the inference proxy "
|
| 224 |
+
"(X-GPU-Energy-J response header). Calls flagged "
|
| 225 |
+
"measured=false fall back to "
|
| 226 |
+
"(data-sheet sustained_power_w Γ duration_s Γ· 3600) "
|
| 227 |
+
"β see app/emissions.HARDWARE for sources. Tokens "
|
| 228 |
+
"are reported by the backend (LiteLLM usage) when "
|
| 229 |
+
"available, else estimated from response text length "
|
| 230 |
+
"(~4 chars/token)."
|
| 231 |
+
),
|
| 232 |
+
}
|
| 233 |
+
|
| 234 |
+
|
| 235 |
+
# Thread-local install. Calls made on threads without an installed
|
| 236 |
+
# tracker hit a no-op stub β always safe to call active().record_*().
|
| 237 |
+
_tl = threading.local()
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
class _NullTracker:
|
| 241 |
+
def record_llm(self, **_kw: Any) -> None:
|
| 242 |
+
return None
|
| 243 |
+
|
| 244 |
+
def record_ml(self, **_kw: Any) -> None:
|
| 245 |
+
return None
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
_NULL = _NullTracker()
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
def install(tracker: Tracker | None) -> None:
|
| 252 |
+
_tl.tracker = tracker
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
def current() -> Tracker | None:
|
| 256 |
+
return getattr(_tl, "tracker", None)
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
def active() -> Tracker | _NullTracker:
|
| 260 |
+
"""Return the installed tracker for this thread, or a no-op stub.
|
| 261 |
+
Always safe to call in instrumentation hot paths."""
|
| 262 |
+
return getattr(_tl, "tracker", None) or _NULL
|
| 263 |
+
|
| 264 |
+
|
| 265 |
+
def estimate_completion_tokens(text: str) -> int:
|
| 266 |
+
"""Rough char/4 estimator used when the backend doesn't report usage
|
| 267 |
+
(e.g. streaming through Ollama, where LiteLLM's stream wrapper does
|
| 268 |
+
not always surface a final usage block)."""
|
| 269 |
+
return max(1, len(text) // 4)
|
app/energy.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Per-query energy footprint estimate.
|
| 2 |
+
|
| 3 |
+
Conservative, defensible numbers β no overclaim. We measure local
|
| 4 |
+
inference time and apply a published-range package-power figure for
|
| 5 |
+
Apple-Silicon LLM inference; we compare to the most recent published
|
| 6 |
+
estimate of frontier-cloud per-query energy (Epoch AI, 2025).
|
| 7 |
+
|
| 8 |
+
This is not a benchmark β it's a transparent rule-of-thumb that the
|
| 9 |
+
user can audit. The system prompt and the UI both surface the
|
| 10 |
+
underlying numbers and the citation.
|
| 11 |
+
"""
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
# Local: Granite 4.1:3b on Apple M-series (M3/M4 Pro range)
|
| 15 |
+
# Sustained package power during ~5 s of LLM inference, q4_K_M quant.
|
| 16 |
+
# Source: ml.energy + community measurements; conservative midpoint.
|
| 17 |
+
LOCAL_PACKAGE_POWER_W = 20.0
|
| 18 |
+
|
| 19 |
+
# Frontier cloud per-query inference energy.
|
| 20 |
+
# Source: Epoch AI, "How much energy does ChatGPT use?" (2025).
|
| 21 |
+
# https://epoch.ai/gradient-updates/how-much-energy-does-chatgpt-use
|
| 22 |
+
# This is a typical-query estimate for GPT-4o-class inference; long-context
|
| 23 |
+
# queries scale roughly linearly with token count.
|
| 24 |
+
CLOUD_PER_QUERY_WH = 0.30
|
| 25 |
+
|
| 26 |
+
# Citation strings used in the UI.
|
| 27 |
+
LOCAL_SOURCE = ("ml.energy / community measurements; ~20 W package power "
|
| 28 |
+
"during Granite 4.1:3b q4_K_M inference on Apple M-series.")
|
| 29 |
+
CLOUD_SOURCE = ('Epoch AI (2025), "How much energy does ChatGPT use?", '
|
| 30 |
+
"estimating ~0.3 Wh per typical GPT-4o query.")
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def estimate(reconcile_seconds: float, total_seconds: float | None = None) -> dict:
|
| 34 |
+
"""Return a per-query energy estimate.
|
| 35 |
+
|
| 36 |
+
Args:
|
| 37 |
+
reconcile_seconds: wallclock of the Granite reconcile step (the
|
| 38 |
+
only step that meaningfully draws CPU/GPU power).
|
| 39 |
+
total_seconds: optional full-FSM wallclock for context.
|
| 40 |
+
"""
|
| 41 |
+
local_wh = LOCAL_PACKAGE_POWER_W * reconcile_seconds / 3600.0
|
| 42 |
+
return {
|
| 43 |
+
"local_wh": round(local_wh, 4),
|
| 44 |
+
"local_mwh": round(local_wh * 1000, 1),
|
| 45 |
+
"cloud_wh": CLOUD_PER_QUERY_WH,
|
| 46 |
+
"cloud_mwh": round(CLOUD_PER_QUERY_WH * 1000, 1),
|
| 47 |
+
"ratio_cloud_over_local": round(CLOUD_PER_QUERY_WH / local_wh, 1) if local_wh > 0 else None,
|
| 48 |
+
"method": {
|
| 49 |
+
"local": f"{LOCAL_PACKAGE_POWER_W} W Γ {reconcile_seconds:.2f} s Γ· 3600",
|
| 50 |
+
"local_source": LOCAL_SOURCE,
|
| 51 |
+
"cloud": f"{CLOUD_PER_QUERY_WH} Wh per query (published estimate)",
|
| 52 |
+
"cloud_source": CLOUD_SOURCE,
|
| 53 |
+
},
|
| 54 |
+
"reconcile_seconds": round(reconcile_seconds, 2),
|
| 55 |
+
"total_seconds": round(total_seconds, 2) if total_seconds is not None else None,
|
| 56 |
+
}
|
app/flood_layers/__init__.py
ADDED
|
File without changes
|
app/flood_layers/dep_stormwater.py
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""NYC DEP Stormwater Flood Maps β pluvial scenarios.
|
| 2 |
+
|
| 3 |
+
Four scenarios, all in EPSG:2263. Polygons are categorized by depth class:
|
| 4 |
+
1 = Nuisance Flooding (>4" and β€1 ft)
|
| 5 |
+
2 = Deep and Contiguous Flooding (>1 ft and β€4 ft)
|
| 6 |
+
3 = Deep Contiguous Flooding (>4 ft)
|
| 7 |
+
|
| 8 |
+
Two query paths exist:
|
| 9 |
+
join_raster(point) β fast path. Samples the baked GeoTIFFs in
|
| 10 |
+
data/baked/. ~3 ms per scenario, ~70 ms cold-open. Used by
|
| 11 |
+
step_dep in the FSM.
|
| 12 |
+
join(assets) β legacy GDB path via gpd.sjoin. Retained as
|
| 13 |
+
a fallback when baked rasters are absent (local dev) and as
|
| 14 |
+
the polygon-overlap path used by coverage_for_polygon for
|
| 15 |
+
neighborhood mode.
|
| 16 |
+
"""
|
| 17 |
+
from __future__ import annotations
|
| 18 |
+
|
| 19 |
+
import logging
|
| 20 |
+
import threading
|
| 21 |
+
from functools import lru_cache
|
| 22 |
+
|
| 23 |
+
import geopandas as gpd
|
| 24 |
+
|
| 25 |
+
from app.spatial import DATA, NYC_CRS
|
| 26 |
+
|
| 27 |
+
log = logging.getLogger(__name__)
|
| 28 |
+
BAKED = DATA / "baked"
|
| 29 |
+
_TLOCAL = threading.local()
|
| 30 |
+
_FALLBACK_WARNED = False
|
| 31 |
+
|
| 32 |
+
ROOT = DATA / "dep"
|
| 33 |
+
|
| 34 |
+
SCENARIOS = {
|
| 35 |
+
"dep_extreme_2080": {
|
| 36 |
+
"gdb": "dep_extreme_2080.gdb",
|
| 37 |
+
"label": "DEP Extreme Stormwater (3.66 in/hr, 2080 SLR)",
|
| 38 |
+
},
|
| 39 |
+
"dep_moderate_2050": {
|
| 40 |
+
"gdb": "dep_moderate_2050.gdb",
|
| 41 |
+
"label": "DEP Moderate Stormwater (2.13 in/hr, 2050 SLR)",
|
| 42 |
+
},
|
| 43 |
+
"dep_moderate_current": {
|
| 44 |
+
"gdb": "dep_moderate_current.gdb",
|
| 45 |
+
"label": "DEP Moderate Stormwater (2.13 in/hr, current SLR)",
|
| 46 |
+
},
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
DEPTH_CLASS = {
|
| 50 |
+
1: "Nuisance (>4 in to 1 ft)",
|
| 51 |
+
2: "Deep & Contiguous (1-4 ft)",
|
| 52 |
+
3: "Deep Contiguous (>4 ft)",
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
@lru_cache(maxsize=4)
|
| 57 |
+
def load(scenario: str) -> gpd.GeoDataFrame:
|
| 58 |
+
s = SCENARIOS[scenario]
|
| 59 |
+
path = ROOT / s["gdb"]
|
| 60 |
+
g = gpd.read_file(str(path))
|
| 61 |
+
if g.crs.to_string() != NYC_CRS:
|
| 62 |
+
g = g.to_crs(NYC_CRS)
|
| 63 |
+
return g
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def join(assets: gpd.GeoDataFrame, scenario: str) -> gpd.GeoDataFrame:
|
| 67 |
+
"""Per-asset depth class, or 0 if outside scenario.
|
| 68 |
+
|
| 69 |
+
Returns a frame indexed like assets with columns: depth_class, depth_label.
|
| 70 |
+
Higher class wins on overlap.
|
| 71 |
+
"""
|
| 72 |
+
z = load(scenario)
|
| 73 |
+
a = assets[["geometry"]].copy()
|
| 74 |
+
a["_aid"] = range(len(a))
|
| 75 |
+
j = gpd.sjoin(a, z[["Flooding_Category", "geometry"]],
|
| 76 |
+
how="left", predicate="intersects")
|
| 77 |
+
# for each asset, take max category hit (3 dominates 1)
|
| 78 |
+
cat = (j.groupby("_aid")["Flooding_Category"].max()
|
| 79 |
+
.reindex(range(len(a)))
|
| 80 |
+
.fillna(0).astype(int))
|
| 81 |
+
out = a[["_aid"]].copy()
|
| 82 |
+
out["depth_class"] = cat.values
|
| 83 |
+
out["depth_label"] = out["depth_class"].map(lambda c: DEPTH_CLASS.get(c, "outside"))
|
| 84 |
+
return out[["depth_class", "depth_label"]].reset_index(drop=True)
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def label(scenario: str) -> str:
|
| 88 |
+
return SCENARIOS[scenario]["label"]
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def _raster_handles():
|
| 92 |
+
"""Per-thread rasterio handle cache. rasterio.DatasetReader is not
|
| 93 |
+
safe to share across threads for concurrent .sample() calls; the
|
| 94 |
+
FSM runs each request on its own executor thread, so we keep one
|
| 95 |
+
handle set per thread."""
|
| 96 |
+
h = getattr(_TLOCAL, "handles", None)
|
| 97 |
+
if h is not None:
|
| 98 |
+
return h
|
| 99 |
+
import rasterio
|
| 100 |
+
h = {}
|
| 101 |
+
for s in SCENARIOS:
|
| 102 |
+
p = BAKED / f"{s}.tif"
|
| 103 |
+
if not p.exists():
|
| 104 |
+
return None
|
| 105 |
+
h[s] = rasterio.open(str(p))
|
| 106 |
+
_TLOCAL.handles = h
|
| 107 |
+
return h
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def join_raster(pt_geom_2263, scenario: str) -> int:
|
| 111 |
+
"""Fast path. Returns the integer depth class (0=outside, 1/2/3) for a
|
| 112 |
+
single shapely Point in EPSG:2263. Falls back to the GDB join() path
|
| 113 |
+
if baked rasters are missing β emits a one-time warning so local dev
|
| 114 |
+
still works without the bake artifacts."""
|
| 115 |
+
global _FALLBACK_WARNED
|
| 116 |
+
h = _raster_handles()
|
| 117 |
+
if h is None:
|
| 118 |
+
if not _FALLBACK_WARNED:
|
| 119 |
+
log.warning(
|
| 120 |
+
"data/baked/dep_*.tif not found β falling back to GDB sjoin. "
|
| 121 |
+
"Run: uv run python scripts/bake_cornerstone_rasters.py"
|
| 122 |
+
)
|
| 123 |
+
_FALLBACK_WARNED = True
|
| 124 |
+
# legacy fallback β wrap point in a one-row GeoDataFrame
|
| 125 |
+
a = gpd.GeoDataFrame(geometry=[pt_geom_2263], crs=NYC_CRS)
|
| 126 |
+
return int(join(a, scenario).iloc[0]["depth_class"])
|
| 127 |
+
ds = h[scenario]
|
| 128 |
+
v = next(ds.sample([(pt_geom_2263.x, pt_geom_2263.y)]))
|
| 129 |
+
return int(v[0])
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def coverage_for_polygon(polygon, scenario: str,
|
| 133 |
+
polygon_crs: str = "EPSG:4326") -> dict:
|
| 134 |
+
"""Polygon-level summary: what fraction of the input polygon falls into
|
| 135 |
+
each depth class for a given DEP scenario? Used in neighborhood mode.
|
| 136 |
+
|
| 137 |
+
Returns:
|
| 138 |
+
{
|
| 139 |
+
'scenario': scenario id,
|
| 140 |
+
'label': human-readable scenario name,
|
| 141 |
+
'fraction_any': fraction of polygon inside any flooded class,
|
| 142 |
+
'fraction_class': {1: f, 2: f, 3: f} fraction in each class,
|
| 143 |
+
'polygon_area_m2': total polygon area,
|
| 144 |
+
}
|
| 145 |
+
"""
|
| 146 |
+
z = load(scenario)
|
| 147 |
+
poly_gdf = gpd.GeoDataFrame(geometry=[polygon], crs=polygon_crs).to_crs(NYC_CRS)
|
| 148 |
+
poly_geom = poly_gdf.iloc[0].geometry
|
| 149 |
+
poly_ft2 = float(poly_geom.area)
|
| 150 |
+
sqft_to_m2 = 0.092903
|
| 151 |
+
fraction_class = {1: 0.0, 2: 0.0, 3: 0.0}
|
| 152 |
+
if poly_ft2:
|
| 153 |
+
for cat in (1, 2, 3):
|
| 154 |
+
sub = z[z["Flooding_Category"] == cat]
|
| 155 |
+
if sub.empty:
|
| 156 |
+
continue
|
| 157 |
+
inter = sub.geometry.intersection(poly_geom)
|
| 158 |
+
inter = inter[~inter.is_empty]
|
| 159 |
+
ft2 = float(inter.area.sum()) if len(inter) else 0.0
|
| 160 |
+
fraction_class[cat] = round(ft2 / poly_ft2, 4)
|
| 161 |
+
fraction_any = round(sum(fraction_class.values()), 4)
|
| 162 |
+
return {
|
| 163 |
+
"scenario": scenario,
|
| 164 |
+
"label": label(scenario),
|
| 165 |
+
"fraction_any": fraction_any,
|
| 166 |
+
"fraction_class": fraction_class,
|
| 167 |
+
"polygon_area_m2": round(poly_ft2 * sqft_to_m2, 1),
|
| 168 |
+
}
|
app/flood_layers/ida_hwm.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Hurricane Ida (Sept 2021) empirical flood extent β USGS high-water marks.
|
| 2 |
+
|
| 3 |
+
This specialist plays the same role as Prithvi-EO 2.0 (Sen1Floods11)
|
| 4 |
+
in the parent triangulation-engine: it provides empirical post-event
|
| 5 |
+
flood evidence (versus the modeled scenarios from FEMA/DEP). Where
|
| 6 |
+
Prithvi derives extent from Sentinel-1 SAR, USGS HWMs are surveyed
|
| 7 |
+
ground-truth water marks. Both are valid empirical signals; HWMs
|
| 8 |
+
are the public record for Ida specifically.
|
| 9 |
+
|
| 10 |
+
Output per address: number of HWMs within radius, max water elevation
|
| 11 |
+
(ft), nearest site description.
|
| 12 |
+
"""
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import json
|
| 16 |
+
import math
|
| 17 |
+
from dataclasses import dataclass
|
| 18 |
+
from functools import lru_cache
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
|
| 21 |
+
DATA = Path(__file__).resolve().parent.parent.parent / "data" / "ida_2021_hwms_ny.geojson"
|
| 22 |
+
DOC_ID = "ida_hwm"
|
| 23 |
+
CITATION = "USGS STN Hurricane Ida 2021 high-water marks (Event 312, NY)"
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
@dataclass
|
| 27 |
+
class HWMSummary:
|
| 28 |
+
n_within_radius: int
|
| 29 |
+
radius_m: int
|
| 30 |
+
max_elev_ft: float | None
|
| 31 |
+
max_height_above_gnd_ft: float | None
|
| 32 |
+
nearest_dist_m: float | None
|
| 33 |
+
nearest_site: str | None
|
| 34 |
+
nearest_elev_ft: float | None
|
| 35 |
+
sample_sites: list[str]
|
| 36 |
+
points: list[dict] | None = None # per-mark for the map layer
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _haversine_m(lat1, lon1, lat2, lon2):
|
| 40 |
+
R = 6371000.0
|
| 41 |
+
p1, p2 = math.radians(lat1), math.radians(lat2)
|
| 42 |
+
dp = math.radians(lat2 - lat1); dl = math.radians(lon2 - lon1)
|
| 43 |
+
a = math.sin(dp / 2) ** 2 + math.cos(p1) * math.cos(p2) * math.sin(dl / 2) ** 2
|
| 44 |
+
return 2 * R * math.asin(math.sqrt(a))
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
@lru_cache(maxsize=1)
|
| 48 |
+
def _load() -> list[dict]:
|
| 49 |
+
if not DATA.exists():
|
| 50 |
+
return []
|
| 51 |
+
with open(DATA) as f:
|
| 52 |
+
return json.load(f).get("features", [])
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def summary_for_point(lat: float, lon: float, radius_m: int = 1000) -> HWMSummary | None:
|
| 56 |
+
feats = _load()
|
| 57 |
+
if not feats:
|
| 58 |
+
return None
|
| 59 |
+
in_radius = []
|
| 60 |
+
nearest = (None, float("inf"), None)
|
| 61 |
+
for f in feats:
|
| 62 |
+
flon, flat = f["geometry"]["coordinates"]
|
| 63 |
+
d = _haversine_m(lat, lon, flat, flon)
|
| 64 |
+
if d <= radius_m:
|
| 65 |
+
in_radius.append((d, f))
|
| 66 |
+
if d < nearest[1]:
|
| 67 |
+
nearest = (f, d, None)
|
| 68 |
+
nf, nd, _ = nearest
|
| 69 |
+
elevs = [f["properties"].get("elev_ft") for _, f in in_radius
|
| 70 |
+
if f["properties"].get("elev_ft") is not None]
|
| 71 |
+
heights = [f["properties"].get("height_above_gnd") for _, f in in_radius
|
| 72 |
+
if f["properties"].get("height_above_gnd") is not None]
|
| 73 |
+
sites = [f["properties"].get("site_description") for _, f in in_radius]
|
| 74 |
+
sites = [s for s in sites if s][:5]
|
| 75 |
+
points = []
|
| 76 |
+
for d, f in in_radius[:50]: # cap so SSE payload stays small
|
| 77 |
+
flon, flat = f["geometry"]["coordinates"]
|
| 78 |
+
p = f["properties"]
|
| 79 |
+
points.append({
|
| 80 |
+
"lat": flat, "lon": flon,
|
| 81 |
+
"site": p.get("site_description"),
|
| 82 |
+
"elev_ft": p.get("elev_ft"),
|
| 83 |
+
"height_above_gnd_ft": p.get("height_above_gnd"),
|
| 84 |
+
"distance_m": round(d, 1),
|
| 85 |
+
})
|
| 86 |
+
return HWMSummary(
|
| 87 |
+
n_within_radius=len(in_radius),
|
| 88 |
+
radius_m=radius_m,
|
| 89 |
+
max_elev_ft=round(max(elevs), 2) if elevs else None,
|
| 90 |
+
max_height_above_gnd_ft=round(max(heights), 2) if heights else None,
|
| 91 |
+
nearest_dist_m=round(nd, 0) if nf is not None else None,
|
| 92 |
+
nearest_site=nf["properties"].get("site_description") if nf else None,
|
| 93 |
+
nearest_elev_ft=nf["properties"].get("elev_ft") if nf else None,
|
| 94 |
+
sample_sites=sites,
|
| 95 |
+
points=points,
|
| 96 |
+
)
|
app/flood_layers/prithvi_live.py
ADDED
|
@@ -0,0 +1,563 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Prithvi-EO 2.0 (NYC Pluvial v2 fine-tune) live water segmentation.
|
| 2 |
+
|
| 3 |
+
A per-query specialist: pulls the most recent low-cloud Sentinel-2 L2A
|
| 4 |
+
scene over the address from Microsoft Planetary Computer, runs the
|
| 5 |
+
NYC-specialized fine-tune, and reports % water within 500 m.
|
| 6 |
+
|
| 7 |
+
Distinct from `app/flood_layers/prithvi_water.py`, which serves the
|
| 8 |
+
offline-precomputed 2021 Ida polygons. This one is *fresh observation*
|
| 9 |
+
each query β same doc_id (`prithvi_live`), but the underlying model
|
| 10 |
+
has been swapped from the Sen1Floods11 base to
|
| 11 |
+
`msradam/Prithvi-EO-2.0-NYC-Pluvial` (Apache-2.0, fine-tuned on AMD
|
| 12 |
+
Instinct MI300X via AMD Developer Cloud β test flood IoU 0.5979,
|
| 13 |
+
6Γ over the base). The base model is still loadable by setting
|
| 14 |
+
RIPRAP_PRITHVI_LIVE_REPO to the IBM repo as a fallback.
|
| 15 |
+
|
| 16 |
+
Network calls (STAC search + COG band reads) and a 300M-param model
|
| 17 |
+
forward pass make this the slowest specialist after the LLM. Gated by
|
| 18 |
+
RIPRAP_PRITHVI_LIVE_ENABLE so deployments without the deps installed
|
| 19 |
+
silently skip it. Cloud-cover refuses out at 30%+ to honor the
|
| 20 |
+
Sen1Floods11 training distribution.
|
| 21 |
+
|
| 22 |
+
License: Apache-2.0. See experiments/shared/licenses.md.
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
from __future__ import annotations
|
| 26 |
+
|
| 27 |
+
import concurrent.futures
|
| 28 |
+
import logging
|
| 29 |
+
import os
|
| 30 |
+
import threading
|
| 31 |
+
import time
|
| 32 |
+
from typing import Any
|
| 33 |
+
|
| 34 |
+
log = logging.getLogger("riprap.prithvi_live")
|
| 35 |
+
|
| 36 |
+
ENABLE = os.environ.get("RIPRAP_PRITHVI_LIVE_ENABLE", "1").lower() in ("1", "true", "yes")
|
| 37 |
+
SEARCH_DAYS = int(os.environ.get("RIPRAP_PRITHVI_LIVE_SEARCH_DAYS", "120"))
|
| 38 |
+
MAX_CLOUD_PCT = float(os.environ.get("RIPRAP_PRITHVI_LIVE_MAX_CLOUD", "30"))
|
| 39 |
+
DEVICE = os.environ.get("RIPRAP_PRITHVI_LIVE_DEVICE", "cpu")
|
| 40 |
+
|
| 41 |
+
# Default to the NYC Pluvial v2 fine-tune; override to the IBM-NASA base
|
| 42 |
+
# (`ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11`) when the v2
|
| 43 |
+
# artifact is unreachable or for A/B comparisons.
|
| 44 |
+
REPO = os.environ.get(
|
| 45 |
+
"RIPRAP_PRITHVI_LIVE_REPO",
|
| 46 |
+
"msradam/Prithvi-EO-2.0-NYC-Pluvial",
|
| 47 |
+
)
|
| 48 |
+
BASE_REPO = "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11"
|
| 49 |
+
|
| 50 |
+
# Sen1Floods11 expects 6 bands in this exact order.
|
| 51 |
+
BANDS = ["B02", "B03", "B04", "B8A", "B11", "B12"]
|
| 52 |
+
IMG_SIZE = 512 # Sen1Floods11 training crop
|
| 53 |
+
CHIP_PX = 1024
|
| 54 |
+
CHIP_M = CHIP_PX * 10
|
| 55 |
+
HALF_M = CHIP_M / 2
|
| 56 |
+
CENTER_RADIUS_M = 500
|
| 57 |
+
PIXEL_M = 10
|
| 58 |
+
|
| 59 |
+
_MODEL = None
|
| 60 |
+
_RUN_MODEL = None
|
| 61 |
+
_INIT_LOCK = threading.Lock() # serializes lazy load if multiple threads
|
| 62 |
+
# hit fetch() before _MODEL is populated
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _has_required_deps() -> tuple[bool, str | None]:
|
| 66 |
+
"""Probe deps in two tiers.
|
| 67 |
+
|
| 68 |
+
Tier 1 β chip fetching (planetary_computer / pystac_client / rioxarray
|
| 69 |
+
/ xarray / einops) is always required: prithvi_live always pulls a
|
| 70 |
+
Sentinel-2 chip from Microsoft Planetary Computer regardless of where
|
| 71 |
+
inference runs.
|
| 72 |
+
|
| 73 |
+
Tier 2 β local inference (terratorch) is only required when remote
|
| 74 |
+
inference is unavailable. On the HF Space we have remote inference
|
| 75 |
+
on the AMD MI300X via app/inference.py, so terratorch is not needed
|
| 76 |
+
even though chip-fetch is.
|
| 77 |
+
|
| 78 |
+
Returns (False, missing) if any required dep is missing. Splitting
|
| 79 |
+
the gate this way lets the HF Space deployment fetch chips and run
|
| 80 |
+
remote inference even though it doesn't fit terratorch's transitive
|
| 81 |
+
dep cone (~250 MB) in the HF build sandbox."""
|
| 82 |
+
chip_deps = ("planetary_computer", "pystac_client",
|
| 83 |
+
"rioxarray", "xarray", "einops")
|
| 84 |
+
missing = [n for n in chip_deps
|
| 85 |
+
if not _has_module(n)]
|
| 86 |
+
if missing:
|
| 87 |
+
return False, ", ".join(missing)
|
| 88 |
+
# Tier 2: only need terratorch if we'd run inference locally.
|
| 89 |
+
try:
|
| 90 |
+
from app import inference as _inf
|
| 91 |
+
if _inf.remote_enabled():
|
| 92 |
+
return True, None
|
| 93 |
+
except Exception:
|
| 94 |
+
pass
|
| 95 |
+
if not _has_module("terratorch"):
|
| 96 |
+
return False, "terratorch (local inference)"
|
| 97 |
+
return True, None
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def _has_module(name: str) -> bool:
|
| 101 |
+
"""True if `name` imports cleanly. ImportError β not installed.
|
| 102 |
+
Other exceptions (e.g. torchvision::nms RuntimeError on the HF
|
| 103 |
+
Space) β treat as unavailable too; we don't want a clean-skip
|
| 104 |
+
intent to crash the FSM at deps-probe time."""
|
| 105 |
+
try:
|
| 106 |
+
__import__(name)
|
| 107 |
+
return True
|
| 108 |
+
except ImportError:
|
| 109 |
+
return False
|
| 110 |
+
except Exception as e:
|
| 111 |
+
log.warning("prithvi_live: %s import raised %s; treating as "
|
| 112 |
+
"unavailable", name, type(e).__name__)
|
| 113 |
+
return False
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
_DEPS_OK, _DEPS_MISSING = _has_required_deps()
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def warm():
|
| 120 |
+
"""Optional pre-load. The FSM action is lazy too β calling warm()
|
| 121 |
+
here just amortizes the first-query cost at app boot."""
|
| 122 |
+
if not ENABLE:
|
| 123 |
+
return
|
| 124 |
+
try:
|
| 125 |
+
_ensure_model()
|
| 126 |
+
except Exception:
|
| 127 |
+
log.exception("prithvi_live: warm() failed; specialist will no-op")
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def _ensure_model():
|
| 131 |
+
"""Load Prithvi-EO 2.0 once into RAM.
|
| 132 |
+
|
| 133 |
+
The v2 NYC Pluvial fine-tune (`msradam/Prithvi-EO-2.0-NYC-Pluvial`)
|
| 134 |
+
is **architecturally distinct** from the IBM-NASA Sen1Floods11
|
| 135 |
+
base: v2 ships a `UNetDecoder` + 2-class head, the base ships a
|
| 136 |
+
UperNet with PSP / FPN. The model has to be built from each
|
| 137 |
+
repo's own config.yaml β there's no key-mapping shim that bridges
|
| 138 |
+
them.
|
| 139 |
+
|
| 140 |
+
Strategy:
|
| 141 |
+
|
| 142 |
+
1. If the active REPO != BASE_REPO, try to build from the v2
|
| 143 |
+
yaml + v2 ckpt. The v2 yaml's data: paths point at the
|
| 144 |
+
training droplet's filesystem (`/root/terramind_nyc/...`)
|
| 145 |
+
which doesn't exist locally; that's fine β the
|
| 146 |
+
GenericNonGeoSegmentationDataModule constructor only
|
| 147 |
+
records the paths, splits aren't read until `setup()`.
|
| 148 |
+
2. On any v2 failure (yaml not present, datamodule constructor
|
| 149 |
+
strict, weights mismatch), fall back to the base yaml + base
|
| 150 |
+
ckpt. The base path is the proven pre-C5 behaviour.
|
| 151 |
+
|
| 152 |
+
The shared `inference.run_model` helper is only published by the
|
| 153 |
+
IBM-NASA base repo; we always pull it from there.
|
| 154 |
+
"""
|
| 155 |
+
global _MODEL, _RUN_MODEL
|
| 156 |
+
if _MODEL is not None:
|
| 157 |
+
return _MODEL, _RUN_MODEL
|
| 158 |
+
with _INIT_LOCK:
|
| 159 |
+
if _MODEL is not None: # double-check inside the lock
|
| 160 |
+
return _MODEL, _RUN_MODEL
|
| 161 |
+
import importlib.util
|
| 162 |
+
|
| 163 |
+
from huggingface_hub import hf_hub_download
|
| 164 |
+
from terratorch.cli_tools import LightningInferenceModel
|
| 165 |
+
log.info("prithvi_live: loading model from %s", REPO)
|
| 166 |
+
|
| 167 |
+
# Inference helper only lives in the IBM-NASA base repo.
|
| 168 |
+
inference_py = hf_hub_download(BASE_REPO, "inference.py")
|
| 169 |
+
|
| 170 |
+
m = None
|
| 171 |
+
# ---- v2 path: yaml + ckpt from the published repo ----------
|
| 172 |
+
if REPO != BASE_REPO:
|
| 173 |
+
try:
|
| 174 |
+
# The v2 repo publishes `prithvi_nyc_phase14.yaml` and
|
| 175 |
+
# `prithvi_nyc_pluvial_v2.ckpt`. Be tolerant of small
|
| 176 |
+
# naming drift (best_val_loss.ckpt etc.) by probing.
|
| 177 |
+
v2_yaml = None
|
| 178 |
+
for name in ("prithvi_nyc_phase14.yaml",
|
| 179 |
+
"config.yaml", "phase14.yaml",
|
| 180 |
+
"prithvi_nyc_v2.yaml"):
|
| 181 |
+
try:
|
| 182 |
+
v2_yaml = hf_hub_download(REPO, name)
|
| 183 |
+
break
|
| 184 |
+
except Exception:
|
| 185 |
+
continue
|
| 186 |
+
v2_ckpt = None
|
| 187 |
+
for name in ("prithvi_nyc_pluvial_v2.ckpt",
|
| 188 |
+
"best_val_loss.ckpt", "model.ckpt",
|
| 189 |
+
"last.ckpt"):
|
| 190 |
+
try:
|
| 191 |
+
v2_ckpt = hf_hub_download(REPO, name)
|
| 192 |
+
break
|
| 193 |
+
except Exception:
|
| 194 |
+
continue
|
| 195 |
+
if v2_yaml and v2_ckpt:
|
| 196 |
+
log.info("prithvi_live: building v2 model from "
|
| 197 |
+
"yaml=%s ckpt=%s", v2_yaml, v2_ckpt)
|
| 198 |
+
m = LightningInferenceModel.from_config(v2_yaml, v2_ckpt)
|
| 199 |
+
# prithvi_nyc_phase14.yaml uses GenericNonGeoSegmentationDataModule
|
| 200 |
+
# which omits test_transform (β None) and uses terratorch Normalize
|
| 201 |
+
# for aug (only handles 4D/5D). IBM inference.py:run_model() calls
|
| 202 |
+
# both on a 3D dict. Patch both to match the IBM base contract:
|
| 203 |
+
# ToTensorV2 for test_transform; Kornia AugmentationSequential
|
| 204 |
+
# (accepts dict input, adds batch dim) for aug.
|
| 205 |
+
if getattr(getattr(m, 'datamodule', None),
|
| 206 |
+
'test_transform', None) is None:
|
| 207 |
+
import albumentations as A
|
| 208 |
+
import torch as _torch
|
| 209 |
+
from albumentations.pytorch import ToTensorV2
|
| 210 |
+
m.datamodule.test_transform = A.Compose([ToTensorV2()])
|
| 211 |
+
_old = m.datamodule.aug
|
| 212 |
+
|
| 213 |
+
# IBM's inference.py:188 calls
|
| 214 |
+
# `datamodule.aug({'image': tensor})['image']`.
|
| 215 |
+
# kornia's AugmentationSequential doesn't accept
|
| 216 |
+
# dict input cleanly and tripped the
|
| 217 |
+
# `'list' object has no attribute 'view'`
|
| 218 |
+
# error on the L4 deploy. Use a hand-rolled
|
| 219 |
+
# dict-aware normalizer instead β same math,
|
| 220 |
+
# fewer moving parts, no kornia version skew.
|
| 221 |
+
class _DictNormalize:
|
| 222 |
+
def __init__(self, mean, std):
|
| 223 |
+
self.mean = _torch.as_tensor(mean).view(-1, 1, 1).float()
|
| 224 |
+
self.std = _torch.as_tensor(std).view(-1, 1, 1).float()
|
| 225 |
+
|
| 226 |
+
def __call__(self, sample):
|
| 227 |
+
if isinstance(sample, dict):
|
| 228 |
+
img = sample["image"]
|
| 229 |
+
mean = self.mean.to(img.device)
|
| 230 |
+
std = self.std.to(img.device)
|
| 231 |
+
return {**sample, "image": (img - mean) / std}
|
| 232 |
+
mean = self.mean.to(sample.device)
|
| 233 |
+
std = self.std.to(sample.device)
|
| 234 |
+
return (sample - mean) / std
|
| 235 |
+
|
| 236 |
+
# `_old.means` / `_old.stds` come from the
|
| 237 |
+
# yaml as Python lists β calling `.view()` on
|
| 238 |
+
# them is what tripped the original
|
| 239 |
+
# `'list' object has no attribute 'view'`.
|
| 240 |
+
# _DictNormalize handles the conversion via
|
| 241 |
+
# torch.as_tensor internally; just pass the
|
| 242 |
+
# raw values whatever their type.
|
| 243 |
+
m.datamodule.aug = _DictNormalize(_old.means, _old.stds)
|
| 244 |
+
log.info("prithvi_live: patched v2 datamodule transforms "
|
| 245 |
+
"for IBM inference.py compat (dict-aware Normalize)")
|
| 246 |
+
else:
|
| 247 |
+
log.warning("prithvi_live: v2 yaml/ckpt not "
|
| 248 |
+
"discoverable in %s; falling back to base",
|
| 249 |
+
REPO)
|
| 250 |
+
except Exception as e:
|
| 251 |
+
log.warning("prithvi_live: v2 build failed (%s); "
|
| 252 |
+
"falling back to base", e)
|
| 253 |
+
m = None
|
| 254 |
+
|
| 255 |
+
# ---- base path: proven IBM-NASA Sen1Floods11 fine-tune -----
|
| 256 |
+
if m is None:
|
| 257 |
+
base_config = hf_hub_download(BASE_REPO, "config.yaml")
|
| 258 |
+
base_ckpt = hf_hub_download(
|
| 259 |
+
BASE_REPO, "Prithvi-EO-V2-300M-TL-Sen1Floods11.pt")
|
| 260 |
+
m = LightningInferenceModel.from_config(base_config, base_ckpt)
|
| 261 |
+
|
| 262 |
+
m.model.eval()
|
| 263 |
+
if DEVICE == "cuda":
|
| 264 |
+
try:
|
| 265 |
+
import torch
|
| 266 |
+
if torch.cuda.is_available():
|
| 267 |
+
m.model.cuda()
|
| 268 |
+
except Exception:
|
| 269 |
+
log.exception("prithvi_live: cuda move failed")
|
| 270 |
+
|
| 271 |
+
spec = importlib.util.spec_from_file_location("_prithvi_inference",
|
| 272 |
+
inference_py)
|
| 273 |
+
mod = importlib.util.module_from_spec(spec)
|
| 274 |
+
spec.loader.exec_module(mod)
|
| 275 |
+
_MODEL = m
|
| 276 |
+
_RUN_MODEL = mod.run_model
|
| 277 |
+
return _MODEL, _RUN_MODEL
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
def _search_recent_scene(lat: float, lon: float):
|
| 281 |
+
"""Most recent low-cloud S2 L2A item near (lat, lon) in the last
|
| 282 |
+
SEARCH_DAYS days, or None."""
|
| 283 |
+
import datetime as dt
|
| 284 |
+
|
| 285 |
+
import planetary_computer as pc
|
| 286 |
+
from pystac_client import Client
|
| 287 |
+
end = dt.datetime.utcnow().date()
|
| 288 |
+
start = end - dt.timedelta(days=SEARCH_DAYS)
|
| 289 |
+
client = Client.open(
|
| 290 |
+
"https://planetarycomputer.microsoft.com/api/stac/v1",
|
| 291 |
+
modifier=pc.sign_inplace,
|
| 292 |
+
)
|
| 293 |
+
delta = 0.02
|
| 294 |
+
search = client.search(
|
| 295 |
+
collections=["sentinel-2-l2a"],
|
| 296 |
+
bbox=[lon - delta, lat - delta, lon + delta, lat + delta],
|
| 297 |
+
datetime=f"{start}/{end}",
|
| 298 |
+
query={"eo:cloud_cover": {"lt": MAX_CLOUD_PCT}},
|
| 299 |
+
max_items=20,
|
| 300 |
+
)
|
| 301 |
+
items = sorted(
|
| 302 |
+
search.items(),
|
| 303 |
+
key=lambda it: (it.properties.get("eo:cloud_cover", 100),
|
| 304 |
+
-(it.datetime.timestamp() if it.datetime else 0)),
|
| 305 |
+
)
|
| 306 |
+
return items[0] if items else None
|
| 307 |
+
|
| 308 |
+
|
| 309 |
+
def _build_chip(item, lat: float, lon: float):
|
| 310 |
+
"""Returns (img, ref_da, epsg) β img is the (6, H, W) center-cropped
|
| 311 |
+
float32 array; ref_da is the rioxarray DataArray of the reference
|
| 312 |
+
band BEFORE the center crop (kept so we can compute the affine
|
| 313 |
+
transform for polygonization in EPSG:4326)."""
|
| 314 |
+
import numpy as np
|
| 315 |
+
import rioxarray # noqa: F401
|
| 316 |
+
import xarray as xr
|
| 317 |
+
from pyproj import Transformer
|
| 318 |
+
if "proj:epsg" in item.properties:
|
| 319 |
+
epsg = int(item.properties["proj:epsg"])
|
| 320 |
+
else:
|
| 321 |
+
code = item.properties.get("proj:code", "")
|
| 322 |
+
if code.startswith("EPSG:"):
|
| 323 |
+
epsg = int(code.split(":", 1)[1])
|
| 324 |
+
else:
|
| 325 |
+
raise RuntimeError("STAC item missing proj:epsg / proj:code")
|
| 326 |
+
fwd = Transformer.from_crs("EPSG:4326", f"EPSG:{epsg}", always_xy=True)
|
| 327 |
+
cx, cy = fwd.transform(lon, lat)
|
| 328 |
+
xmin, xmax = cx - HALF_M, cx + HALF_M
|
| 329 |
+
ymin, ymax = cy - HALF_M, cy + HALF_M
|
| 330 |
+
ref = rioxarray.open_rasterio(item.assets[BANDS[0]].href, masked=False).squeeze(drop=True)
|
| 331 |
+
ref = ref.rio.clip_box(minx=xmin, miny=ymin, maxx=xmax, maxy=ymax)
|
| 332 |
+
ref = ref.isel(y=slice(0, CHIP_PX), x=slice(0, CHIP_PX))
|
| 333 |
+
arrs = [ref.astype("float32")]
|
| 334 |
+
for b in BANDS[1:]:
|
| 335 |
+
da = rioxarray.open_rasterio(item.assets[b].href, masked=False).squeeze(drop=True)
|
| 336 |
+
da = da.rio.clip_box(minx=xmin, miny=ymin, maxx=xmax, maxy=ymax)
|
| 337 |
+
if da.shape != ref.shape:
|
| 338 |
+
da = da.rio.reproject_match(ref)
|
| 339 |
+
arrs.append(da.astype("float32"))
|
| 340 |
+
stacked = xr.concat(arrs, dim="band", join="override").assign_coords(band=BANDS)
|
| 341 |
+
img = stacked.values # (6, H, W)
|
| 342 |
+
# Center crop to IMG_SIZE x IMG_SIZE.
|
| 343 |
+
_, h, w = img.shape
|
| 344 |
+
sy, sx = (h - IMG_SIZE) // 2, (w - IMG_SIZE) // 2
|
| 345 |
+
img = img[:, sy:sy + IMG_SIZE, sx:sx + IMG_SIZE]
|
| 346 |
+
if img.mean() > 1:
|
| 347 |
+
img = img / 10000.0
|
| 348 |
+
return np.nan_to_num(img.astype("float32")), ref, epsg
|
| 349 |
+
|
| 350 |
+
|
| 351 |
+
def _polygonize_mask(pred, ref_da, epsg: int) -> dict | None:
|
| 352 |
+
"""Vectorize the binary water mask into an EPSG:4326 GeoJSON
|
| 353 |
+
FeatureCollection so the frontend can paint it on the MapLibre
|
| 354 |
+
map. Returns None on failure (best-effort β never raises into the
|
| 355 |
+
caller path)."""
|
| 356 |
+
try:
|
| 357 |
+
import json
|
| 358 |
+
|
| 359 |
+
import geopandas as gpd
|
| 360 |
+
from rasterio.features import shapes
|
| 361 |
+
from rasterio.transform import from_origin
|
| 362 |
+
from shapely.geometry import shape
|
| 363 |
+
# Reconstruct the affine transform of the center-cropped pred.
|
| 364 |
+
# ref_da has 1024 px at 10 m; we cropped to the central 512.
|
| 365 |
+
xs = ref_da.x.values
|
| 366 |
+
ys = ref_da.y.values
|
| 367 |
+
if len(xs) < IMG_SIZE or len(ys) < IMG_SIZE:
|
| 368 |
+
return None
|
| 369 |
+
# rioxarray gives pixel-centered coords; offset by half a pixel
|
| 370 |
+
# to the upper-left to build a from_origin transform.
|
| 371 |
+
sy = (len(ys) - IMG_SIZE) // 2
|
| 372 |
+
sx = (len(xs) - IMG_SIZE) // 2
|
| 373 |
+
# ys are descending (top-to-bottom); take the top of the crop.
|
| 374 |
+
top_y = float(ys[sy]) + (PIXEL_M / 2.0)
|
| 375 |
+
left_x = float(xs[sx]) - (PIXEL_M / 2.0)
|
| 376 |
+
transform = from_origin(left_x, top_y, PIXEL_M, PIXEL_M)
|
| 377 |
+
# Polygonize only the water class (1).
|
| 378 |
+
mask = (pred == 1).astype("uint8")
|
| 379 |
+
polys = []
|
| 380 |
+
for geom, value in shapes(mask, mask=mask.astype(bool),
|
| 381 |
+
transform=transform):
|
| 382 |
+
if value != 1:
|
| 383 |
+
continue
|
| 384 |
+
polys.append(shape(geom))
|
| 385 |
+
if not polys:
|
| 386 |
+
return {"type": "FeatureCollection", "features": []}
|
| 387 |
+
gdf = gpd.GeoDataFrame({"geometry": polys},
|
| 388 |
+
crs=f"EPSG:{epsg}").to_crs("EPSG:4326")
|
| 389 |
+
# Simplify slightly to keep the SSE payload small (10 m raster
|
| 390 |
+
# over 5 km square = up to ~10 k tiny squares; simplification
|
| 391 |
+
# collapses adjacent water pixels into smooth polygons).
|
| 392 |
+
gdf["geometry"] = gdf.geometry.simplify(0.00005, preserve_topology=True)
|
| 393 |
+
return json.loads(gdf.to_json())
|
| 394 |
+
except Exception:
|
| 395 |
+
log.exception("prithvi_live: polygonize failed")
|
| 396 |
+
return None
|
| 397 |
+
|
| 398 |
+
|
| 399 |
+
def _fetch_inner(lat: float, lon: float, timeout_s: float) -> dict[str, Any]:
|
| 400 |
+
"""Core fetch logic β run inside a bounded thread via fetch()."""
|
| 401 |
+
t0 = time.time()
|
| 402 |
+
try:
|
| 403 |
+
item = _search_recent_scene(lat, lon)
|
| 404 |
+
if item is None:
|
| 405 |
+
return {"ok": False, "skipped": f"no <{MAX_CLOUD_PCT}% cloud "
|
| 406 |
+
f"S2 in last {SEARCH_DAYS}d"}
|
| 407 |
+
cc = float(item.properties.get("eo:cloud_cover", -1))
|
| 408 |
+
if time.time() - t0 > timeout_s:
|
| 409 |
+
return {"ok": False, "skipped": "stac search exceeded budget"}
|
| 410 |
+
img, ref_da, epsg = _build_chip(item, lat, lon)
|
| 411 |
+
if time.time() - t0 > timeout_s:
|
| 412 |
+
return {"ok": False, "skipped": "chip build exceeded budget"}
|
| 413 |
+
|
| 414 |
+
# v0.4.5 β try the MI300X inference service first if configured.
|
| 415 |
+
# On RemoteUnreachable (service down / not configured / 5xx) fall
|
| 416 |
+
# through to the local terratorch path. When remote is configured
|
| 417 |
+
# but returns non-ok we surface that signal directly: the local
|
| 418 |
+
# path on this machine has been brittle (v2 datamodule
|
| 419 |
+
# `test_transform=None` race), so a configured remote is more
|
| 420 |
+
# reliable than the fallback.
|
| 421 |
+
remote_attempted = False
|
| 422 |
+
try:
|
| 423 |
+
from app import inference as _inf
|
| 424 |
+
if _inf.remote_enabled():
|
| 425 |
+
remote_attempted = True
|
| 426 |
+
remote = _inf.prithvi_pluvial(
|
| 427 |
+
img, scene_id=item.id,
|
| 428 |
+
scene_datetime=str(item.datetime),
|
| 429 |
+
cloud_cover=cc,
|
| 430 |
+
timeout=timeout_s,
|
| 431 |
+
)
|
| 432 |
+
if remote.get("ok"):
|
| 433 |
+
# Vectorize the remote prediction raster so the map
|
| 434 |
+
# actually renders the live water polygons. The
|
| 435 |
+
# droplet returns `pred_b64` (uint8 binary mask);
|
| 436 |
+
# we polygonize against the chip's WGS84 bounds
|
| 437 |
+
# which we know locally from `ref_da`.
|
| 438 |
+
polys = None
|
| 439 |
+
pred_b64 = remote.get("pred_b64")
|
| 440 |
+
pred_shape = remote.get("pred_shape")
|
| 441 |
+
if pred_b64 and pred_shape:
|
| 442 |
+
try:
|
| 443 |
+
xs = ref_da.x.values
|
| 444 |
+
ys = ref_da.y.values
|
| 445 |
+
from pyproj import Transformer
|
| 446 |
+
t_inv = Transformer.from_crs(
|
| 447 |
+
f"EPSG:{epsg}", "EPSG:4326",
|
| 448 |
+
always_xy=True)
|
| 449 |
+
minx, maxx = float(xs.min()), float(xs.max())
|
| 450 |
+
miny, maxy = float(ys.min()), float(ys.max())
|
| 451 |
+
minlon, minlat = t_inv.transform(minx, miny)
|
| 452 |
+
maxlon, maxlat = t_inv.transform(maxx, maxy)
|
| 453 |
+
from app.context._polygonize import (
|
| 454 |
+
polygonize_binary_mask,
|
| 455 |
+
)
|
| 456 |
+
polys = polygonize_binary_mask(
|
| 457 |
+
pred_b64, pred_shape,
|
| 458 |
+
(minlon, minlat, maxlon, maxlat),
|
| 459 |
+
label="water", fill_color="#1F77B4",
|
| 460 |
+
simplify_tolerance=2e-5,
|
| 461 |
+
)
|
| 462 |
+
except Exception:
|
| 463 |
+
log.exception("prithvi_live: remote polygonize failed")
|
| 464 |
+
polys = None
|
| 465 |
+
return {
|
| 466 |
+
"ok": True,
|
| 467 |
+
"item_id": item.id,
|
| 468 |
+
"item_datetime": str(item.datetime),
|
| 469 |
+
"cloud_cover": cc,
|
| 470 |
+
"pct_water_full": remote.get("pct_water_full"),
|
| 471 |
+
"pct_water_within_500m": remote.get("pct_water_within_500m"),
|
| 472 |
+
"polygons_geojson": polys,
|
| 473 |
+
"compute": f"remote Β· {remote.get('device', 'gpu')}",
|
| 474 |
+
"elapsed_s": round(time.time() - t0, 2),
|
| 475 |
+
}
|
| 476 |
+
err = (remote.get("err")
|
| 477 |
+
or remote.get("error")
|
| 478 |
+
or remote.get("skipped")
|
| 479 |
+
or "unknown")
|
| 480 |
+
return {"ok": False,
|
| 481 |
+
"skipped": f"remote prithvi-pluvial non-ok: {err}",
|
| 482 |
+
"elapsed_s": round(time.time() - t0, 2)}
|
| 483 |
+
except _inf.RemoteUnreachable as e:
|
| 484 |
+
log.info("prithvi_live: remote unreachable (%s)", e)
|
| 485 |
+
if remote_attempted:
|
| 486 |
+
# Don't fall to local β torchvision::nms is broken on the
|
| 487 |
+
# CPU-tier UI Spaces and crashes the FSM specialist with
|
| 488 |
+
# a confusing RuntimeError. Return a clean skipped row so
|
| 489 |
+
# the trace says "remote unreachable" instead.
|
| 490 |
+
return {"ok": False,
|
| 491 |
+
"skipped": f"remote prithvi-pluvial unreachable: {e}",
|
| 492 |
+
"elapsed_s": round(time.time() - t0, 2)}
|
| 493 |
+
except Exception as e:
|
| 494 |
+
log.exception("prithvi_live: remote call failed")
|
| 495 |
+
if remote_attempted:
|
| 496 |
+
return {"ok": False,
|
| 497 |
+
"skipped": f"remote prithvi-pluvial error: "
|
| 498 |
+
f"{type(e).__name__}: {e}",
|
| 499 |
+
"elapsed_s": round(time.time() - t0, 2)}
|
| 500 |
+
|
| 501 |
+
# Local fallback β the path that's been live since v0.4.4.
|
| 502 |
+
# Reached only when remote_attempted is False (i.e. remote
|
| 503 |
+
# backend not configured at all).
|
| 504 |
+
model, run_model = _ensure_model()
|
| 505 |
+
x = img[None, :, None, :, :] # (1, 6, 1, H, W)
|
| 506 |
+
pred_t = run_model(x, None, None, model.model, model.datamodule, IMG_SIZE)
|
| 507 |
+
import numpy as np
|
| 508 |
+
pred = pred_t[0].cpu().numpy().astype("uint8")
|
| 509 |
+
pct_full = float(100.0 * pred.mean())
|
| 510 |
+
yy, xx = np.indices(pred.shape)
|
| 511 |
+
cy, cx = pred.shape[0] // 2, pred.shape[1] // 2
|
| 512 |
+
radius_px = CENTER_RADIUS_M / PIXEL_M
|
| 513 |
+
circle = (yy - cy) ** 2 + (xx - cx) ** 2 <= radius_px ** 2
|
| 514 |
+
pct_500 = float(100.0 * pred[circle].mean()) if circle.sum() else 0.0
|
| 515 |
+
polygons_geojson = _polygonize_mask(pred, ref_da, epsg)
|
| 516 |
+
return {
|
| 517 |
+
"ok": True,
|
| 518 |
+
"item_id": item.id,
|
| 519 |
+
"item_datetime": str(item.datetime),
|
| 520 |
+
"cloud_cover": cc,
|
| 521 |
+
"pct_water_full": pct_full,
|
| 522 |
+
"pct_water_within_500m": pct_500,
|
| 523 |
+
"polygons_geojson": polygons_geojson,
|
| 524 |
+
"compute": "local",
|
| 525 |
+
"elapsed_s": round(time.time() - t0, 2),
|
| 526 |
+
}
|
| 527 |
+
except Exception as e:
|
| 528 |
+
log.exception("prithvi_live: fetch failed")
|
| 529 |
+
return {"ok": False, "err": f"{type(e).__name__}: {e}",
|
| 530 |
+
"elapsed_s": round(time.time() - t0, 2)}
|
| 531 |
+
|
| 532 |
+
|
| 533 |
+
def fetch(lat: float, lon: float, timeout_s: float = 60.0) -> dict[str, Any]:
|
| 534 |
+
"""Run the specialist. Wraps _fetch_inner in a bounded thread so that
|
| 535 |
+
STAC searches and COG band reads (which lack per-request HTTP timeouts)
|
| 536 |
+
cannot hang the FSM indefinitely.
|
| 537 |
+
|
| 538 |
+
Returns a dict with at minimum:
|
| 539 |
+
{ "ok": bool, "skipped": str | None, "item_id": str | None,
|
| 540 |
+
"cloud_cover": float | None, "pct_water_within_500m": float | None }
|
| 541 |
+
Designed to never raise; failures show up as ok=False with an `err`.
|
| 542 |
+
"""
|
| 543 |
+
if not ENABLE:
|
| 544 |
+
return {"ok": False, "skipped": "RIPRAP_PRITHVI_LIVE_ENABLE=0"}
|
| 545 |
+
if not _DEPS_OK:
|
| 546 |
+
return {"ok": False,
|
| 547 |
+
"skipped": f"deps unavailable on this deployment: "
|
| 548 |
+
f"{_DEPS_MISSING}"}
|
| 549 |
+
hard_timeout = timeout_s + 15.0
|
| 550 |
+
from app import emissions as _emissions
|
| 551 |
+
_parent_tracker = _emissions.current()
|
| 552 |
+
with concurrent.futures.ThreadPoolExecutor(
|
| 553 |
+
max_workers=1,
|
| 554 |
+
initializer=lambda t=_parent_tracker: _emissions.install(t),
|
| 555 |
+
) as pool:
|
| 556 |
+
future = pool.submit(_fetch_inner, lat, lon, timeout_s)
|
| 557 |
+
try:
|
| 558 |
+
return future.result(timeout=hard_timeout)
|
| 559 |
+
except concurrent.futures.TimeoutError:
|
| 560 |
+
log.warning("prithvi_live: hard timeout after %.0fs (STAC/COG hung)",
|
| 561 |
+
hard_timeout)
|
| 562 |
+
return {"ok": False,
|
| 563 |
+
"skipped": f"prithvi_live timed out after {hard_timeout:.0f}s"}
|
app/flood_layers/prithvi_water.py
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Prithvi-EO 2.0 (Sen1Floods11) satellite flood inundation specialist.
|
| 2 |
+
|
| 3 |
+
The 300M-parameter Prithvi-EO foundation model (NASA/IBM, Apache-2.0)
|
| 4 |
+
was run twice offline on Hurricane Ida 2021 pre/post HLS Sentinel-2
|
| 5 |
+
scenes over central NYC:
|
| 6 |
+
|
| 7 |
+
pre : HLS.S30.T18TWK.2021237T153809 (2021-08-25, 3% cloud)
|
| 8 |
+
post: HLS.S30.T18TWK.2021245T154911 (2021-09-02, 1% cloud,
|
| 9 |
+
~12 hours after peak rainfall)
|
| 10 |
+
|
| 11 |
+
The diff (post-water minus pre-water, filtered to β₯3-cell polygons)
|
| 12 |
+
isolates surface water present 12 hours after Ida that wasn't present
|
| 13 |
+
the prior week β i.e., candidate Ida-attributable inundation. We ship
|
| 14 |
+
the resulting polygons as a flood-layer specialist; per query we
|
| 15 |
+
compute proximity from the address to the nearest such polygon.
|
| 16 |
+
|
| 17 |
+
Honest scope:
|
| 18 |
+
- Sub-surface flooding (subway entrances, basement apartments β the
|
| 19 |
+
dominant Ida damage mode in NYC) is not visible to optical satellites.
|
| 20 |
+
- Pluvial street water had largely drained by the Sep 2 16:02Z pass,
|
| 21 |
+
so the residual Prithvi signal mostly captures marsh ponding,
|
| 22 |
+
riverside spillover, and low-lying park inundation.
|
| 23 |
+
- The model fired on Ida itself (a real flood event), not a synthetic
|
| 24 |
+
fallback β that's the architectural value.
|
| 25 |
+
"""
|
| 26 |
+
from __future__ import annotations
|
| 27 |
+
|
| 28 |
+
import json
|
| 29 |
+
import math
|
| 30 |
+
from dataclasses import dataclass
|
| 31 |
+
from functools import lru_cache
|
| 32 |
+
from pathlib import Path
|
| 33 |
+
|
| 34 |
+
DATA_DIR = Path(__file__).resolve().parent.parent.parent / "data"
|
| 35 |
+
DOC_ID = "prithvi_water"
|
| 36 |
+
CITATION = ("Prithvi-EO-2.0-300M-TL-Sen1Floods11 (NASA/IBM, Apache-2.0, via "
|
| 37 |
+
"TerraTorch). Hurricane Ida pre/post diff: pre HLS T18TWK "
|
| 38 |
+
"2021-08-25 (3% cloud), post HLS T18TWK 2021-09-02 (1% cloud, "
|
| 39 |
+
"~12h after peak rainfall).")
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
@dataclass
|
| 43 |
+
class PrithviSummary:
|
| 44 |
+
inside_water_polygon: bool
|
| 45 |
+
nearest_distance_m: float | None
|
| 46 |
+
n_polygons_within_500m: int
|
| 47 |
+
scene_id: str
|
| 48 |
+
scene_date: str
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def _haversine_m(lat1, lon1, lat2, lon2):
|
| 52 |
+
R = 6371000.0
|
| 53 |
+
p1, p2 = math.radians(lat1), math.radians(lat2)
|
| 54 |
+
dp = math.radians(lat2 - lat1); dl = math.radians(lon2 - lon1)
|
| 55 |
+
a = math.sin(dp / 2) ** 2 + math.cos(p1) * math.cos(p2) * math.sin(dl / 2) ** 2
|
| 56 |
+
return 2 * R * math.asin(math.sqrt(a))
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
@lru_cache(maxsize=1)
|
| 60 |
+
def _load():
|
| 61 |
+
"""Load the merged Prithvi water mask (combined across NYC MGRS tiles)
|
| 62 |
+
as a GeoDataFrame in NYC state plane (EPSG:2263) for fast metric
|
| 63 |
+
distance queries."""
|
| 64 |
+
import geopandas as gpd
|
| 65 |
+
# Prefer the Ida flood-event diff (real flood-attribution signal);
|
| 66 |
+
# fall back to clear-day permanent-water masks if the Ida file is absent.
|
| 67 |
+
candidates = [
|
| 68 |
+
DATA_DIR / "prithvi_ida_2021.geojson",
|
| 69 |
+
DATA_DIR / "prithvi_flood_nyc.geojson",
|
| 70 |
+
]
|
| 71 |
+
candidates += sorted(DATA_DIR.glob("prithvi_flood_*.geojson"), reverse=True)
|
| 72 |
+
path = next((p for p in candidates if p.exists()), None)
|
| 73 |
+
if path is None:
|
| 74 |
+
return None, None
|
| 75 |
+
with open(path) as f:
|
| 76 |
+
meta = json.load(f)
|
| 77 |
+
g = gpd.read_file(path)
|
| 78 |
+
if g.crs is None:
|
| 79 |
+
g.set_crs("EPSG:4326", inplace=True)
|
| 80 |
+
g = g.to_crs("EPSG:2263")
|
| 81 |
+
return g, meta
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def warm() -> None:
|
| 85 |
+
_load()
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def summary_for_point(lat: float, lon: float) -> PrithviSummary | None:
|
| 89 |
+
import geopandas as gpd
|
| 90 |
+
from shapely.geometry import Point
|
| 91 |
+
g, meta = _load()
|
| 92 |
+
if g is None:
|
| 93 |
+
return None
|
| 94 |
+
pt_wgs = gpd.GeoSeries([Point(lon, lat)], crs="EPSG:4326")
|
| 95 |
+
pt_2263 = pt_wgs.to_crs("EPSG:2263").iloc[0]
|
| 96 |
+
inside = bool(g.contains(pt_2263).any())
|
| 97 |
+
|
| 98 |
+
# nearest distance (feet -> metres)
|
| 99 |
+
distances_ft = g.geometry.distance(pt_2263)
|
| 100 |
+
nearest_ft = float(distances_ft.min()) if len(distances_ft) else None
|
| 101 |
+
nearest_m = round(nearest_ft / 3.281, 1) if nearest_ft is not None else None
|
| 102 |
+
|
| 103 |
+
within_500m = int((distances_ft <= 500 * 3.281).sum())
|
| 104 |
+
|
| 105 |
+
# The Ida pre/post artifact carries pre_/post_ scene info; the clear-day
|
| 106 |
+
# artifact carries scene_ids[]. Format compactly for either case.
|
| 107 |
+
if "post_scene_id" in meta:
|
| 108 |
+
sid = f"pre {meta['pre_scene_id']} | post {meta['post_scene_id']}"
|
| 109 |
+
sdate = f"pre {meta['pre_scene_date']}, post {meta['post_scene_date']}"
|
| 110 |
+
else:
|
| 111 |
+
sid = meta.get("scene_id") or ", ".join(meta.get("scene_ids", []) or ["unknown"])
|
| 112 |
+
sdate = meta.get("scene_date") or ", ".join(meta.get("scene_dates", []) or ["unknown"])
|
| 113 |
+
|
| 114 |
+
return PrithviSummary(
|
| 115 |
+
inside_water_polygon=inside,
|
| 116 |
+
nearest_distance_m=nearest_m,
|
| 117 |
+
n_polygons_within_500m=within_500m,
|
| 118 |
+
scene_id=sid,
|
| 119 |
+
scene_date=sdate,
|
| 120 |
+
)
|
app/flood_layers/sandy_inundation.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""NYC Sandy Inundation Zone (empirical 2012 extent, NYC OD 5xsi-dfpx).
|
| 2 |
+
|
| 3 |
+
Two query paths exist:
|
| 4 |
+
inside_raster(point) β fast path. Samples data/baked/sandy.tif.
|
| 5 |
+
~1 ms; used by step_sandy in the FSM.
|
| 6 |
+
join(assets) β legacy GeoJSON sjoin path. Retained as a
|
| 7 |
+
fallback when the baked raster is absent (local dev) and
|
| 8 |
+
for coverage_for_polygon (neighborhood mode).
|
| 9 |
+
"""
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import logging
|
| 13 |
+
import threading
|
| 14 |
+
from functools import lru_cache
|
| 15 |
+
|
| 16 |
+
import geopandas as gpd
|
| 17 |
+
|
| 18 |
+
from app.spatial import DATA, NYC_CRS, load_layer
|
| 19 |
+
|
| 20 |
+
DOC_ID = "sandy_inundation"
|
| 21 |
+
CITATION = "NYC Sandy Inundation Zone (NYC OpenData 5xsi-dfpx, empirical 2012 extent)"
|
| 22 |
+
|
| 23 |
+
log = logging.getLogger(__name__)
|
| 24 |
+
BAKED = DATA / "baked"
|
| 25 |
+
_TLOCAL = threading.local()
|
| 26 |
+
_FALLBACK_WARNED = False
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
@lru_cache(maxsize=1)
|
| 30 |
+
def load() -> gpd.GeoDataFrame:
|
| 31 |
+
g = load_layer(DATA / "sandy_inundation.geojson")
|
| 32 |
+
return g[["geometry"]]
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def join(assets: gpd.GeoDataFrame) -> gpd.pd.Series:
|
| 36 |
+
"""Return a boolean Series indexed like assets: True if inside Sandy zone."""
|
| 37 |
+
z = load()
|
| 38 |
+
# spatial join avoids fragile unary union over messy public polygons
|
| 39 |
+
hits = gpd.sjoin(
|
| 40 |
+
assets[["geometry"]].assign(_aid=range(len(assets))),
|
| 41 |
+
z[["geometry"]],
|
| 42 |
+
how="left",
|
| 43 |
+
predicate="intersects",
|
| 44 |
+
)
|
| 45 |
+
flagged = hits.dropna(subset=["index_right"])["_aid"].unique()
|
| 46 |
+
s = assets.geometry.copy().astype(bool)
|
| 47 |
+
s[:] = False
|
| 48 |
+
s.iloc[list(flagged)] = True
|
| 49 |
+
return s.reset_index(drop=True)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _raster_handle():
|
| 53 |
+
"""Per-thread rasterio handle. See dep_stormwater._raster_handles."""
|
| 54 |
+
h = getattr(_TLOCAL, "handle", None)
|
| 55 |
+
if h is not None:
|
| 56 |
+
return h
|
| 57 |
+
p = BAKED / "sandy.tif"
|
| 58 |
+
if not p.exists():
|
| 59 |
+
return None
|
| 60 |
+
import rasterio
|
| 61 |
+
h = rasterio.open(str(p))
|
| 62 |
+
_TLOCAL.handle = h
|
| 63 |
+
return h
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def inside_raster(pt_geom_2263) -> bool:
|
| 67 |
+
"""Fast path. True if the shapely Point (in EPSG:2263) falls inside the
|
| 68 |
+
2012 Sandy inundation extent. Falls back to the GeoJSON sjoin path if
|
| 69 |
+
data/baked/sandy.tif is missing."""
|
| 70 |
+
global _FALLBACK_WARNED
|
| 71 |
+
h = _raster_handle()
|
| 72 |
+
if h is None:
|
| 73 |
+
if not _FALLBACK_WARNED:
|
| 74 |
+
log.warning(
|
| 75 |
+
"data/baked/sandy.tif not found β falling back to GeoJSON sjoin. "
|
| 76 |
+
"Run: uv run python scripts/bake_cornerstone_rasters.py"
|
| 77 |
+
)
|
| 78 |
+
_FALLBACK_WARNED = True
|
| 79 |
+
a = gpd.GeoDataFrame(geometry=[pt_geom_2263], crs=NYC_CRS)
|
| 80 |
+
return bool(join(a).iloc[0])
|
| 81 |
+
v = next(h.sample([(pt_geom_2263.x, pt_geom_2263.y)]))
|
| 82 |
+
return bool(int(v[0]))
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def coverage_for_polygon(polygon, polygon_crs: str = "EPSG:4326") -> dict:
|
| 86 |
+
"""Polygon-level summary: what fraction of the input polygon overlaps
|
| 87 |
+
the 2012 Sandy inundation extent? Used in neighborhood-mode queries.
|
| 88 |
+
|
| 89 |
+
Returns:
|
| 90 |
+
{
|
| 91 |
+
'overlap_area_m2': absolute overlap in m2,
|
| 92 |
+
'polygon_area_m2': total polygon area in m2,
|
| 93 |
+
'fraction': overlap / polygon_area, range [0, 1],
|
| 94 |
+
'inside': True if any overlap exists,
|
| 95 |
+
}
|
| 96 |
+
"""
|
| 97 |
+
z = load().to_crs("EPSG:2263") # NY State Plane Long Island, units = ft
|
| 98 |
+
poly_gdf = gpd.GeoDataFrame(geometry=[polygon], crs=polygon_crs).to_crs("EPSG:2263")
|
| 99 |
+
poly_geom = poly_gdf.iloc[0].geometry
|
| 100 |
+
inter = z.intersection(poly_geom)
|
| 101 |
+
inter = inter[~inter.is_empty]
|
| 102 |
+
overlap_ft2 = float(inter.area.sum()) if len(inter) else 0.0
|
| 103 |
+
poly_ft2 = float(poly_geom.area)
|
| 104 |
+
sqft_to_m2 = 0.092903
|
| 105 |
+
return {
|
| 106 |
+
"overlap_area_m2": round(overlap_ft2 * sqft_to_m2, 1),
|
| 107 |
+
"polygon_area_m2": round(poly_ft2 * sqft_to_m2, 1),
|
| 108 |
+
"fraction": round(overlap_ft2 / poly_ft2, 4) if poly_ft2 else 0.0,
|
| 109 |
+
"inside": overlap_ft2 > 0,
|
| 110 |
+
}
|
app/framing.py
ADDED
|
@@ -0,0 +1,249 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Question-aware framing for the Capstone briefing opening.
|
| 2 |
+
|
| 3 |
+
The four-section structure (Status / Empirical / Modeled / Policy) is
|
| 4 |
+
load-bearing for the Mellea grounding checks and stays unchanged. What
|
| 5 |
+
this module does is detect the *shape* of the user's question from the
|
| 6 |
+
raw query string + planner intent, then return a single-sentence
|
| 7 |
+
directive that conditions only the opening Status sentence.
|
| 8 |
+
|
| 9 |
+
Eleven question types are recognised; they mirror the rubric in
|
| 10 |
+
`tests/integration/stakeholder_queries.py:FRAMING_RUBRICS`. Detection
|
| 11 |
+
is deterministic regex matching β no extra LLM call, no added latency.
|
| 12 |
+
|
| 13 |
+
Usage:
|
| 14 |
+
|
| 15 |
+
from app.framing import augment_system_prompt
|
| 16 |
+
system_prompt = augment_system_prompt(
|
| 17 |
+
EXTRA_SYSTEM_PROMPT, query=user_query, intent=plan.intent,
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
The returned prompt has the original text plus a trailing
|
| 21 |
+
`QUESTION-AWARE OPENING:` block. Granite 4.1 attends to this through
|
| 22 |
+
the system-prompt cache and applies it to the Status sentence.
|
| 23 |
+
"""
|
| 24 |
+
from __future__ import annotations
|
| 25 |
+
|
| 26 |
+
import re
|
| 27 |
+
from typing import Final
|
| 28 |
+
|
| 29 |
+
QUESTION_TYPES: Final[tuple[str, ...]] = (
|
| 30 |
+
"habitability_decision",
|
| 31 |
+
"legal_disclosure",
|
| 32 |
+
"capital_planning",
|
| 33 |
+
"underwriting",
|
| 34 |
+
"journalism",
|
| 35 |
+
"development_siting",
|
| 36 |
+
"grant_evidence",
|
| 37 |
+
"retrospective",
|
| 38 |
+
"emergency_response",
|
| 39 |
+
"comparison",
|
| 40 |
+
"generic_exposure",
|
| 41 |
+
)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
# ---- Per-type opening directives ------------------------------------------
|
| 45 |
+
#
|
| 46 |
+
# Each directive is one sentence that supplements (does not replace) the
|
| 47 |
+
# Status section's existing instruction. Granite 4.1 has a strong prior
|
| 48 |
+
# toward "this address is exposed to ..." openings; the directive
|
| 49 |
+
# overrides that in a question-shaped way without disturbing the four
|
| 50 |
+
# grounding invariants.
|
| 51 |
+
|
| 52 |
+
_DIRECTIVES: dict[str, str] = {
|
| 53 |
+
"habitability_decision": (
|
| 54 |
+
"The Status sentence MUST start with a direct verdict word "
|
| 55 |
+
"(\"Yes\" if the documents show meaningful flood evidence, \"No\" "
|
| 56 |
+
"if they don't), then name the single strongest piece of "
|
| 57 |
+
"evidence with its [doc_id]. The user is deciding whether to "
|
| 58 |
+
"live here β answer the question, then cite."
|
| 59 |
+
),
|
| 60 |
+
"legal_disclosure": (
|
| 61 |
+
"The Status sentence MUST state whether the documents contain "
|
| 62 |
+
"facts a NY RPL Β§462(2) or Β§231-b disclosure would need to "
|
| 63 |
+
"record. Begin with \"Disclosure is warranted\" or \"Disclosure "
|
| 64 |
+
"is not triggered\" based on the evidence, then name the "
|
| 65 |
+
"specific fact with its [doc_id]. The user is a real-estate "
|
| 66 |
+
"professional checking the disclosure threshold."
|
| 67 |
+
),
|
| 68 |
+
"capital_planning": (
|
| 69 |
+
"The Status sentence MUST frame the place as a capital-planning "
|
| 70 |
+
"candidate: name the dominant exposure with its [doc_id] and "
|
| 71 |
+
"indicate whether the evidence supports prioritization "
|
| 72 |
+
"(\"merits prioritization\", \"ranks high for hardening\") or "
|
| 73 |
+
"not. The user allocates infrastructure investment."
|
| 74 |
+
),
|
| 75 |
+
"underwriting": (
|
| 76 |
+
"The Status sentence MUST emphasize that every figure in the "
|
| 77 |
+
"briefing is independently sourced β open with the dominant "
|
| 78 |
+
"exposure and the specific [doc_id], then add a half-clause "
|
| 79 |
+
"noting that the audit chain follows below. The user is an "
|
| 80 |
+
"underwriter who needs a defensible loss narrative."
|
| 81 |
+
),
|
| 82 |
+
"journalism": (
|
| 83 |
+
"The Status sentence MUST be reproducible reporting prose: "
|
| 84 |
+
"name the place, name the dominant exposure with [doc_id], "
|
| 85 |
+
"and avoid editorial verbs like \"shocking\" or \"alarming\". "
|
| 86 |
+
"The user is a data journalist who will cite this prose verbatim."
|
| 87 |
+
),
|
| 88 |
+
"development_siting": (
|
| 89 |
+
"The Status sentence MUST start with the count of active "
|
| 90 |
+
"construction filings cited from [dob_permits] (e.g. \"N "
|
| 91 |
+
"active construction filings sit inside ...\") and indicate "
|
| 92 |
+
"which flood layer they intersect. The user is a developer or "
|
| 93 |
+
"architect doing a pre-design siting check."
|
| 94 |
+
),
|
| 95 |
+
"grant_evidence": (
|
| 96 |
+
"The Status sentence MUST open with \"Vulnerability "
|
| 97 |
+
"assessment:\" and name the place + dominant exposure with "
|
| 98 |
+
"[doc_id]. Treat the briefing as the evidence section of a "
|
| 99 |
+
"HUD CDBG-DR or FEMA BRIC application β formal, third-person, "
|
| 100 |
+
"free of advocacy framing."
|
| 101 |
+
),
|
| 102 |
+
"retrospective": (
|
| 103 |
+
"Riprap currently runs on present-day data sources. The Status "
|
| 104 |
+
"sentence MUST acknowledge the question is retrospective and "
|
| 105 |
+
"state explicitly that the briefing reflects the CURRENT state "
|
| 106 |
+
"of these data sources, not a snapshot from the requested date. "
|
| 107 |
+
"Then proceed with the present-day exposure picture so the user "
|
| 108 |
+
"still gets the geography. Silence-over-confabulation: never "
|
| 109 |
+
"reconstruct historical conditions you can't verify."
|
| 110 |
+
),
|
| 111 |
+
"emergency_response": (
|
| 112 |
+
"The Status sentence MUST quantify what is at risk in the "
|
| 113 |
+
"next few hours, citing the live signal that triggered the "
|
| 114 |
+
"query and any active alerts with [doc_id]. The user needs an "
|
| 115 |
+
"operational picture, not a historical exposure summary."
|
| 116 |
+
),
|
| 117 |
+
"comparison": (
|
| 118 |
+
"The Status sentence MUST name BOTH places the user is "
|
| 119 |
+
"comparing and indicate which one shows greater exposure on "
|
| 120 |
+
"the strongest cited signal. If only one place's data is "
|
| 121 |
+
"available in the documents, say so explicitly. The user is "
|
| 122 |
+
"doing a head-to-head decision."
|
| 123 |
+
),
|
| 124 |
+
"generic_exposure": "", # default β no override
|
| 125 |
+
}
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
# ---- Detector -------------------------------------------------------------
|
| 129 |
+
#
|
| 130 |
+
# Patterns are ordered: the FIRST type whose pattern matches wins. Order
|
| 131 |
+
# matters β more specific question shapes (legal_disclosure, grant_evidence,
|
| 132 |
+
# emergency_response) come before more general ones (habitability_decision,
|
| 133 |
+
# capital_planning) so the obvious specialist tags don't get swallowed.
|
| 134 |
+
|
| 135 |
+
_PATTERNS: list[tuple[str, list[re.Pattern]]] = [
|
| 136 |
+
("retrospective", [
|
| 137 |
+
re.compile(r"\b(would have|would Riprap|on (the )?date of|as of (the )?(date|day)|"
|
| 138 |
+
r"day before|prior to|before (Hurricane|Ida|Sandy|the storm)|"
|
| 139 |
+
r"on (August|September|October|November|December|January|February|March|"
|
| 140 |
+
r"April|May|June|July) \d{1,2},? ?\d{4}|"
|
| 141 |
+
r"time.?machine|retrospective|court (exhibit|testimony))\b", re.I),
|
| 142 |
+
]),
|
| 143 |
+
("emergency_response", [
|
| 144 |
+
re.compile(r"\b(just triggered|right now|next (few |six |\d+ )?hours?|"
|
| 145 |
+
r"in the next \d+|currently flooding|flood (warning|watch) is active|"
|
| 146 |
+
r"sensor [A-Z]{2}-?\d+|live (alert|trigger))\b", re.I),
|
| 147 |
+
]),
|
| 148 |
+
("legal_disclosure", [
|
| 149 |
+
re.compile(r"\b(disclos(e|ure|ed)|RPL\s*Β§?\s*\d+|Property Condition Disclosure|"
|
| 150 |
+
r"Β§\s*462|Β§\s*231-?b|seller'?s? disclosure|landlord'?s? disclosure|"
|
| 151 |
+
r"required to disclose|need to disclose)\b", re.I),
|
| 152 |
+
]),
|
| 153 |
+
("grant_evidence", [
|
| 154 |
+
re.compile(r"\b(vulnerability assessment|CDBG-?DR|HUD|BRIC|"
|
| 155 |
+
r"grant application|funding application|community resilience grant|"
|
| 156 |
+
r"FEMA application|disaster recovery (application|funding))\b", re.I),
|
| 157 |
+
]),
|
| 158 |
+
("development_siting", [
|
| 159 |
+
re.compile(r"\b(what (are|is) (they|being) build(ing)?|new construction|"
|
| 160 |
+
r"under construction|active (construction|filing|project|permit)|"
|
| 161 |
+
r"projects? (in progress|underway|planned)|architects?|"
|
| 162 |
+
r"siting check|pre.?design|"
|
| 163 |
+
r"DOB filing|developer)\b", re.I),
|
| 164 |
+
]),
|
| 165 |
+
("comparison", [
|
| 166 |
+
# `prioritize X over Y` can have many words between, hence the
|
| 167 |
+
# bounded non-greedy span β capped at 80 chars to avoid runaway.
|
| 168 |
+
re.compile(r"\b(compare\b|comparison|\bvs\b|\bversus\b|"
|
| 169 |
+
r"head-?to-?head|\brank\s+the\s+top)\b", re.I),
|
| 170 |
+
re.compile(r"\bprioritize\b.{1,80}\bover\b", re.I | re.S),
|
| 171 |
+
re.compile(r"\bover\s+\w+(?:\s+\w+){0,3}\s+for\s+(hardening|investment)\b", re.I),
|
| 172 |
+
]),
|
| 173 |
+
("capital_planning", [
|
| 174 |
+
re.compile(r"\b(prioritiz(e|ation)|capital plan(ning)?|harden(ing|s)?|"
|
| 175 |
+
r"infrastructure investment|where (should|to) (we |the )(invest|"
|
| 176 |
+
r"prioritize|harden)|MTA.+prioritize|DEP.+prioritize|"
|
| 177 |
+
r"protection envelope|outside (it|the protection)|"
|
| 178 |
+
r"resilien(ce|cy) project)\b", re.I),
|
| 179 |
+
]),
|
| 180 |
+
("habitability_decision", [
|
| 181 |
+
re.compile(r"\b(should I worry|should I (be|consider)|is (it|this) safe|"
|
| 182 |
+
r"can I (rent|live|move|raise (my )?kids?)|considering (renting|leasing|moving)|"
|
| 183 |
+
r"(thinking about|planning to) (rent|lease|move|buy)|"
|
| 184 |
+
r"is (this|that|the landlord) true|landlord (says|claims|told)|"
|
| 185 |
+
r"no flood history|just got a lease|new lease|signing a lease|"
|
| 186 |
+
r"\bworry\b)", re.I),
|
| 187 |
+
]),
|
| 188 |
+
("underwriting", [
|
| 189 |
+
re.compile(r"\b(underwrit(e|er|ing|able)|actuarial|loss history|"
|
| 190 |
+
r"insurabl[ey]|catastrophe (model|risk)|"
|
| 191 |
+
r"insurance (audit|memo|profile)|"
|
| 192 |
+
r"audit (chain|trail))\b", re.I),
|
| 193 |
+
]),
|
| 194 |
+
("journalism", [
|
| 195 |
+
re.compile(r"\b(reporter|journalist|newsroom|story|coverage|"
|
| 196 |
+
r"published?|publish (this|the))", re.I),
|
| 197 |
+
]),
|
| 198 |
+
]
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
def detect(query: str, intent: str | None = None) -> str:
|
| 202 |
+
"""Classify the question shape from the raw query and planner intent.
|
| 203 |
+
|
| 204 |
+
Returns one of `QUESTION_TYPES`. Falls back to `generic_exposure`
|
| 205 |
+
when no pattern matches β that's the existing behavior, preserved.
|
| 206 |
+
|
| 207 |
+
`intent` is currently advisory only (the patterns don't read it),
|
| 208 |
+
but the parameter is part of the API so future refinements can
|
| 209 |
+
use it (e.g. an `intent=neighborhood` query without a verdict
|
| 210 |
+
keyword could default to `journalism` rather than `generic_exposure`).
|
| 211 |
+
"""
|
| 212 |
+
if not query:
|
| 213 |
+
return "generic_exposure"
|
| 214 |
+
q = query.strip()
|
| 215 |
+
for qt, patterns in _PATTERNS:
|
| 216 |
+
if any(p.search(q) for p in patterns):
|
| 217 |
+
return qt
|
| 218 |
+
# Heuristic fallback: bare neighborhood/borough names from a planner
|
| 219 |
+
# context default to journalism (most common stakeholder reading a
|
| 220 |
+
# neighborhood-only query is a reporter or planner). For
|
| 221 |
+
# single_address with no question keyword, fall back to generic.
|
| 222 |
+
if intent == "neighborhood" and len(q.split()) <= 3:
|
| 223 |
+
return "journalism"
|
| 224 |
+
return "generic_exposure"
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
def opening_instruction(question_type: str) -> str:
|
| 228 |
+
"""Return the directive sentence(s) for a question type.
|
| 229 |
+
Returns empty string for `generic_exposure` (no override)."""
|
| 230 |
+
return _DIRECTIVES.get(question_type, "")
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
def augment_system_prompt(base: str, *, query: str,
|
| 234 |
+
intent: str | None = None) -> str:
|
| 235 |
+
"""Wrap a base system prompt with a question-aware opening directive.
|
| 236 |
+
|
| 237 |
+
No-op when the detector returns `generic_exposure` β the original
|
| 238 |
+
behavior is preserved.
|
| 239 |
+
"""
|
| 240 |
+
qt = detect(query, intent)
|
| 241 |
+
directive = opening_instruction(qt)
|
| 242 |
+
if not directive:
|
| 243 |
+
return base
|
| 244 |
+
return (
|
| 245 |
+
f"{base}\n\n"
|
| 246 |
+
f"QUESTION-AWARE OPENING (this directive overrides ONLY the opening "
|
| 247 |
+
f"**Status.** sentence; the four-section structure and citation "
|
| 248 |
+
f"discipline above remain in force):\n{directive}"
|
| 249 |
+
)
|
app/fsm.py
ADDED
|
@@ -0,0 +1,1394 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Riprap Burr FSM β linear specialist pipeline for one address.
|
| 2 |
+
|
| 3 |
+
Each action either produces a structured fact (which becomes a document
|
| 4 |
+
the reconciler can cite) or stays silent on failure. The reconciler
|
| 5 |
+
(Granite 4.1) only sees documents from specialists that actually
|
| 6 |
+
produced data β the silence-over-confabulation contract.
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import logging
|
| 11 |
+
import threading as _threading
|
| 12 |
+
import time
|
| 13 |
+
from typing import Any
|
| 14 |
+
|
| 15 |
+
import geopandas as gpd
|
| 16 |
+
from burr.core import ApplicationBuilder, State, action
|
| 17 |
+
from shapely.geometry import Point
|
| 18 |
+
|
| 19 |
+
from app import emissions
|
| 20 |
+
from app.context import floodnet, microtopo, noaa_tides, npcc4_slr, nws_alerts, nws_obs, nyc311
|
| 21 |
+
from app.energy import estimate as energy_estimate
|
| 22 |
+
from app.flood_layers import dep_stormwater, ida_hwm, prithvi_water, sandy_inundation
|
| 23 |
+
from app.geocode import geocode_one
|
| 24 |
+
from app.live import floodnet_forecast as fn_forecast
|
| 25 |
+
from app.live import ttm_forecast
|
| 26 |
+
from app.rag import retrieve as rag_retrieve
|
| 27 |
+
from app.reconcile import citations_from_docs, reconcile as run_reconcile
|
| 28 |
+
from app.registers import doe_schools as r_schools
|
| 29 |
+
from app.registers import doh_hospitals as r_hospitals
|
| 30 |
+
from app.registers import mta_entrances as r_mta
|
| 31 |
+
from app.registers import nycha as r_nycha
|
| 32 |
+
|
| 33 |
+
log = logging.getLogger("riprap.fsm")
|
| 34 |
+
|
| 35 |
+
# NYC five-borough bbox. Specialists whose data sources are NYC-only
|
| 36 |
+
# (Sandy 2012, NYC DEP Stormwater, FloodNet, NYC 311, NYC microtopo
|
| 37 |
+
# raster, NYC Hurricane Ida Prithvi polygons) skip with an explicit
|
| 38 |
+
# "out of NYC scope" reason when geocode lands outside this envelope.
|
| 39 |
+
# Live specialists (NWS / NOAA / TTM) and the NY-State Ida HWMs run
|
| 40 |
+
# unconditionally.
|
| 41 |
+
_NYC_S, _NYC_W, _NYC_N, _NYC_E = 40.49, -74.27, 40.92, -73.69
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def _in_nyc(lat, lon) -> bool:
|
| 45 |
+
if lat is None or lon is None:
|
| 46 |
+
return False
|
| 47 |
+
return _NYC_S <= lat <= _NYC_N and _NYC_W <= lon <= _NYC_E
|
| 48 |
+
|
| 49 |
+
# Thread-local hook so the streaming endpoint can subscribe to per-token
|
| 50 |
+
# Granite output during reconcile, without threading a callback through
|
| 51 |
+
# every Burr action signature.
|
| 52 |
+
_FSM_LOCAL = _threading.local()
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def set_token_callback(on_token):
|
| 56 |
+
"""Install a per-thread on_token(delta) callable for the next reconcile.
|
| 57 |
+
Pass None to clear."""
|
| 58 |
+
_FSM_LOCAL.on_token = on_token
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _current_token_callback():
|
| 62 |
+
return getattr(_FSM_LOCAL, "on_token", None)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def set_mellea_attempt_callback(fn):
|
| 66 |
+
_FSM_LOCAL.on_mellea_attempt = fn
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def _current_mellea_attempt_callback():
|
| 70 |
+
return getattr(_FSM_LOCAL, "on_mellea_attempt", None)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def set_strict_mode(strict: bool):
|
| 74 |
+
"""Per-thread flag β when True the linear FSM's reconcile step routes
|
| 75 |
+
through Mellea-validated rejection sampling instead of the standard
|
| 76 |
+
streaming reconciler. Disables token streaming for that step."""
|
| 77 |
+
_FSM_LOCAL.strict = bool(strict)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def _current_strict_mode() -> bool:
|
| 81 |
+
return bool(getattr(_FSM_LOCAL, "strict", False))
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def set_planned_specialists(spec_names):
|
| 85 |
+
"""Install a per-thread set of specialist names from the planner.
|
| 86 |
+
|
| 87 |
+
Used by step_reconcile to trim doc messages: documents whose family
|
| 88 |
+
prefix doesn't match any planned specialist are dropped before the
|
| 89 |
+
Mellea call. Cuts ~30-50% of prompt tokens on local Ollama, where
|
| 90 |
+
the FSM otherwise hands the reconciler every specialist's output
|
| 91 |
+
even if the planner only asked for a subset."""
|
| 92 |
+
_FSM_LOCAL.planned_specialists = set(spec_names) if spec_names else None
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def _current_planned_specialists():
|
| 96 |
+
return getattr(_FSM_LOCAL, "planned_specialists", None)
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def set_user_query(query: str | None):
|
| 100 |
+
"""Install the user's original natural-language query for question-aware
|
| 101 |
+
framing in step_reconcile. The FSM's state["query"] is the geocoder
|
| 102 |
+
input (often just the street address), which doesn't carry the
|
| 103 |
+
user's question shape β set this separately so Capstone can detect
|
| 104 |
+
'should I worry' / 'is disclosure required' / etc."""
|
| 105 |
+
_FSM_LOCAL.user_query = query
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def _current_user_query() -> str | None:
|
| 109 |
+
return getattr(_FSM_LOCAL, "user_query", None)
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def set_planner_intent(intent: str | None):
|
| 113 |
+
"""Install the planner's classified intent so step_reconcile can pass
|
| 114 |
+
it to the framing detector as a tiebreaker on bare-place queries."""
|
| 115 |
+
_FSM_LOCAL.planner_intent = intent
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def _current_planner_intent() -> str | None:
|
| 119 |
+
return getattr(_FSM_LOCAL, "planner_intent", None)
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
# Canonical Burr: one action per specialist, sequential transitions.
|
| 123 |
+
# A previous version of this module wrapped 16 specialists in a single
|
| 124 |
+
# fan-out action that ran them concurrently in a ThreadPoolExecutor;
|
| 125 |
+
# that path was removed because it sometimes hung after the fan-out
|
| 126 |
+
# completed (Burr-internal post-action cleanup with custom executors)
|
| 127 |
+
# and made the trace UI's per-step timing harder to reason about.
|
| 128 |
+
# Parallelism, when wanted, belongs at the inference layer
|
| 129 |
+
# (vLLM / Ollama NUM_PARALLEL), not the FSM.
|
| 130 |
+
|
| 131 |
+
def _step(state: State, name: str) -> dict[str, Any]:
|
| 132 |
+
"""Append a step record to the trace; returns the dict so the action
|
| 133 |
+
can mutate timing/result fields."""
|
| 134 |
+
trace = list(state.get("trace", []))
|
| 135 |
+
rec = {"step": name, "started_at": time.time(), "ok": None}
|
| 136 |
+
trace.append(rec)
|
| 137 |
+
return rec, trace
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
@action(reads=["query"], writes=["geocode", "lat", "lon", "trace"])
|
| 141 |
+
def step_geocode(state: State) -> State:
|
| 142 |
+
rec, trace = _step(state, "geocode")
|
| 143 |
+
try:
|
| 144 |
+
hit = geocode_one(state["query"])
|
| 145 |
+
if hit is None:
|
| 146 |
+
rec["ok"] = False
|
| 147 |
+
rec["err"] = "no geocoder match"
|
| 148 |
+
# Burr requires every declared write to be populated. Emit
|
| 149 |
+
# explicit None rather than leaving keys absent.
|
| 150 |
+
return state.update(geocode=None, lat=None, lon=None, trace=trace)
|
| 151 |
+
rec["ok"] = True
|
| 152 |
+
rec["result"] = {"address": hit.address, "lat": hit.lat, "lon": hit.lon}
|
| 153 |
+
return state.update(
|
| 154 |
+
geocode={"address": hit.address, "borough": hit.borough,
|
| 155 |
+
"lat": hit.lat, "lon": hit.lon,
|
| 156 |
+
"bbl": hit.bbl, "bin": hit.bin},
|
| 157 |
+
lat=hit.lat, lon=hit.lon, trace=trace,
|
| 158 |
+
)
|
| 159 |
+
except Exception as e:
|
| 160 |
+
rec["ok"] = False
|
| 161 |
+
rec["err"] = str(e)
|
| 162 |
+
log.exception("geocode failed")
|
| 163 |
+
return state.update(geocode=None, lat=None, lon=None, trace=trace)
|
| 164 |
+
finally:
|
| 165 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
@action(reads=["lat", "lon"], writes=["sandy", "trace"])
|
| 169 |
+
def step_sandy(state: State) -> State:
|
| 170 |
+
rec, trace = _step(state, "sandy_inundation")
|
| 171 |
+
try:
|
| 172 |
+
if state.get("lat") is None:
|
| 173 |
+
rec["ok"] = False; rec["err"] = "no coords"
|
| 174 |
+
return state.update(sandy=None, trace=trace)
|
| 175 |
+
if not _in_nyc(state["lat"], state["lon"]):
|
| 176 |
+
rec["ok"] = False; rec["err"] = "out of NYC scope"
|
| 177 |
+
return state.update(sandy=None, trace=trace)
|
| 178 |
+
pt_geom = (gpd.GeoDataFrame(geometry=[Point(state["lon"], state["lat"])],
|
| 179 |
+
crs="EPSG:4326")
|
| 180 |
+
.to_crs("EPSG:2263").iloc[0].geometry)
|
| 181 |
+
flag = sandy_inundation.inside_raster(pt_geom)
|
| 182 |
+
rec["ok"] = True; rec["result"] = {"inside": flag}
|
| 183 |
+
return state.update(sandy=flag, trace=trace)
|
| 184 |
+
except Exception as e:
|
| 185 |
+
rec["ok"] = False; rec["err"] = str(e)
|
| 186 |
+
log.exception("sandy failed")
|
| 187 |
+
return state.update(sandy=None, trace=trace)
|
| 188 |
+
finally:
|
| 189 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
@action(reads=["lat", "lon"], writes=["dep", "trace"])
|
| 193 |
+
def step_dep(state: State) -> State:
|
| 194 |
+
rec, trace = _step(state, "dep_stormwater")
|
| 195 |
+
try:
|
| 196 |
+
if state.get("lat") is None:
|
| 197 |
+
rec["ok"] = False; rec["err"] = "no coords"
|
| 198 |
+
return state.update(dep=None, trace=trace)
|
| 199 |
+
if not _in_nyc(state["lat"], state["lon"]):
|
| 200 |
+
rec["ok"] = False; rec["err"] = "out of NYC scope"
|
| 201 |
+
return state.update(dep=None, trace=trace)
|
| 202 |
+
pt_geom = (gpd.GeoDataFrame(geometry=[Point(state["lon"], state["lat"])],
|
| 203 |
+
crs="EPSG:4326")
|
| 204 |
+
.to_crs("EPSG:2263").iloc[0].geometry)
|
| 205 |
+
out: dict[str, Any] = {}
|
| 206 |
+
for scen in ["dep_extreme_2080", "dep_moderate_2050", "dep_moderate_current"]:
|
| 207 |
+
cls = dep_stormwater.join_raster(pt_geom, scen)
|
| 208 |
+
out[scen] = {
|
| 209 |
+
"depth_class": cls,
|
| 210 |
+
"depth_label": dep_stormwater.DEPTH_CLASS.get(cls, "outside"),
|
| 211 |
+
"citation": f"NYC DEP Stormwater Flood Map β {dep_stormwater.label(scen)}",
|
| 212 |
+
}
|
| 213 |
+
rec["ok"] = True; rec["result"] = {k: v["depth_label"] for k, v in out.items()}
|
| 214 |
+
return state.update(dep=out, trace=trace)
|
| 215 |
+
except Exception as e:
|
| 216 |
+
rec["ok"] = False; rec["err"] = str(e)
|
| 217 |
+
log.exception("dep failed")
|
| 218 |
+
return state.update(dep=None, trace=trace)
|
| 219 |
+
finally:
|
| 220 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
@action(reads=["lat", "lon"], writes=["floodnet", "trace"])
|
| 224 |
+
def step_floodnet(state: State) -> State:
|
| 225 |
+
rec, trace = _step(state, "floodnet")
|
| 226 |
+
try:
|
| 227 |
+
if state.get("lat") is None:
|
| 228 |
+
rec["ok"] = False; rec["err"] = "no coords"
|
| 229 |
+
return state.update(floodnet=None, trace=trace)
|
| 230 |
+
if not _in_nyc(state["lat"], state["lon"]):
|
| 231 |
+
rec["ok"] = False; rec["err"] = "out of NYC scope"
|
| 232 |
+
return state.update(floodnet=None, trace=trace)
|
| 233 |
+
s = floodnet.summary_for_point(state["lat"], state["lon"], radius_m=600)
|
| 234 |
+
s["radius_m"] = 600
|
| 235 |
+
rec["ok"] = True
|
| 236 |
+
rec["result"] = {"n_sensors": s["n_sensors"],
|
| 237 |
+
"n_events_3y": s["n_flood_events_3y"]}
|
| 238 |
+
return state.update(floodnet=s, trace=trace)
|
| 239 |
+
except Exception as e:
|
| 240 |
+
rec["ok"] = False; rec["err"] = str(e)
|
| 241 |
+
log.exception("floodnet failed")
|
| 242 |
+
return state.update(floodnet=None, trace=trace)
|
| 243 |
+
finally:
|
| 244 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
@action(reads=["lat", "lon"], writes=["nyc311", "trace"])
|
| 248 |
+
def step_311(state: State) -> State:
|
| 249 |
+
rec, trace = _step(state, "nyc311")
|
| 250 |
+
try:
|
| 251 |
+
if state.get("lat") is None:
|
| 252 |
+
rec["ok"] = False; rec["err"] = "no coords"
|
| 253 |
+
return state.update(nyc311=None, trace=trace)
|
| 254 |
+
if not _in_nyc(state["lat"], state["lon"]):
|
| 255 |
+
rec["ok"] = False; rec["err"] = "out of NYC scope"
|
| 256 |
+
return state.update(nyc311=None, trace=trace)
|
| 257 |
+
s = nyc311.summary_for_point(state["lat"], state["lon"], radius_m=200, years=5)
|
| 258 |
+
rec["ok"] = True; rec["result"] = {"n": s["n"]}
|
| 259 |
+
return state.update(nyc311=s, trace=trace)
|
| 260 |
+
except Exception as e:
|
| 261 |
+
rec["ok"] = False; rec["err"] = str(e)
|
| 262 |
+
log.exception("311 failed")
|
| 263 |
+
return state.update(nyc311=None, trace=trace)
|
| 264 |
+
finally:
|
| 265 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
@action(reads=["lat", "lon"], writes=["ida_hwm", "trace"])
|
| 269 |
+
def step_ida_hwm(state: State) -> State:
|
| 270 |
+
rec, trace = _step(state, "ida_hwm_2021")
|
| 271 |
+
try:
|
| 272 |
+
if state.get("lat") is None:
|
| 273 |
+
rec["ok"] = False; rec["err"] = "no coords"
|
| 274 |
+
return state.update(ida_hwm=None, trace=trace)
|
| 275 |
+
s = ida_hwm.summary_for_point(state["lat"], state["lon"], radius_m=800)
|
| 276 |
+
if s is None:
|
| 277 |
+
rec["ok"] = False; rec["err"] = "HWM data missing"
|
| 278 |
+
return state.update(ida_hwm=None, trace=trace)
|
| 279 |
+
rec["ok"] = True
|
| 280 |
+
rec["result"] = {
|
| 281 |
+
"n_within_800m": s.n_within_radius,
|
| 282 |
+
"max_height_above_gnd_ft": s.max_height_above_gnd_ft,
|
| 283 |
+
"nearest_m": s.nearest_dist_m,
|
| 284 |
+
}
|
| 285 |
+
return state.update(ida_hwm=vars(s), trace=trace)
|
| 286 |
+
except Exception as e:
|
| 287 |
+
rec["ok"] = False; rec["err"] = str(e)
|
| 288 |
+
log.exception("ida_hwm failed")
|
| 289 |
+
return state.update(ida_hwm=None, trace=trace)
|
| 290 |
+
finally:
|
| 291 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 292 |
+
|
| 293 |
+
|
| 294 |
+
@action(reads=["lat", "lon"], writes=["prithvi_water", "trace"])
|
| 295 |
+
def step_prithvi(state: State) -> State:
|
| 296 |
+
rec, trace = _step(state, "prithvi_eo_v2")
|
| 297 |
+
try:
|
| 298 |
+
if state.get("lat") is None:
|
| 299 |
+
rec["ok"] = False; rec["err"] = "no coords"
|
| 300 |
+
return state.update(prithvi_water=None, trace=trace)
|
| 301 |
+
if not _in_nyc(state["lat"], state["lon"]):
|
| 302 |
+
rec["ok"] = False; rec["err"] = "out of NYC scope"
|
| 303 |
+
return state.update(prithvi_water=None, trace=trace)
|
| 304 |
+
s = prithvi_water.summary_for_point(state["lat"], state["lon"])
|
| 305 |
+
if s is None:
|
| 306 |
+
rec["ok"] = False; rec["err"] = "Prithvi mask missing"
|
| 307 |
+
return state.update(prithvi_water=None, trace=trace)
|
| 308 |
+
rec["ok"] = True
|
| 309 |
+
rec["result"] = {
|
| 310 |
+
"inside_water_polygon": s.inside_water_polygon,
|
| 311 |
+
"nearest_distance_m": s.nearest_distance_m,
|
| 312 |
+
"n_polygons_within_500m": s.n_polygons_within_500m,
|
| 313 |
+
}
|
| 314 |
+
return state.update(prithvi_water=vars(s), trace=trace)
|
| 315 |
+
except Exception as e:
|
| 316 |
+
rec["ok"] = False; rec["err"] = str(e)
|
| 317 |
+
log.exception("prithvi failed")
|
| 318 |
+
return state.update(prithvi_water=None, trace=trace)
|
| 319 |
+
finally:
|
| 320 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 321 |
+
|
| 322 |
+
|
| 323 |
+
@action(reads=["lat", "lon"], writes=["prithvi_live", "trace"])
|
| 324 |
+
def step_prithvi_live(state: State) -> State:
|
| 325 |
+
"""Live Sentinel-2 water segmentation via Prithvi-EO 2.0.
|
| 326 |
+
|
| 327 |
+
Network + 300M-param forward pass per query, so it's the slowest
|
| 328 |
+
specialist by far. Gracefully no-ops via the underlying module if
|
| 329 |
+
`RIPRAP_PRITHVI_LIVE_ENABLE=0` or if STAC / model load fails.
|
| 330 |
+
"""
|
| 331 |
+
rec, trace = _step(state, "prithvi_eo_live")
|
| 332 |
+
try:
|
| 333 |
+
if state.get("lat") is None:
|
| 334 |
+
rec["ok"] = False; rec["err"] = "no coords"
|
| 335 |
+
return state.update(prithvi_live=None, trace=trace)
|
| 336 |
+
if not _in_nyc(state["lat"], state["lon"]):
|
| 337 |
+
rec["ok"] = False; rec["err"] = "out of NYC scope"
|
| 338 |
+
return state.update(prithvi_live=None, trace=trace)
|
| 339 |
+
from app.flood_layers import prithvi_live
|
| 340 |
+
s = prithvi_live.fetch(state["lat"], state["lon"])
|
| 341 |
+
rec["ok"] = bool(s.get("ok"))
|
| 342 |
+
if not s.get("ok"):
|
| 343 |
+
rec["err"] = s.get("err") or s.get("skipped") or "no observation"
|
| 344 |
+
else:
|
| 345 |
+
rec["result"] = {
|
| 346 |
+
"scene_date": (s.get("item_datetime") or "")[:10],
|
| 347 |
+
"cloud_cover": s.get("cloud_cover"),
|
| 348 |
+
"pct_water_500m": s.get("pct_water_within_500m"),
|
| 349 |
+
"pct_water_5km": s.get("pct_water_full"),
|
| 350 |
+
}
|
| 351 |
+
return state.update(prithvi_live=s, trace=trace)
|
| 352 |
+
except Exception as e:
|
| 353 |
+
rec["ok"] = False; rec["err"] = str(e)
|
| 354 |
+
log.exception("prithvi_live failed")
|
| 355 |
+
return state.update(prithvi_live=None, trace=trace)
|
| 356 |
+
finally:
|
| 357 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 358 |
+
|
| 359 |
+
|
| 360 |
+
@action(reads=["lat", "lon"], writes=["ttm_311_forecast", "trace"])
|
| 361 |
+
def step_ttm_311_forecast(state: State) -> State:
|
| 362 |
+
"""TTM r2 zero-shot forecast on weekly 311 flood-complaint counts
|
| 363 |
+
at this specific address (200 m radius). 52 weeks of context β
|
| 364 |
+
4 weeks of forecast. Per-query, per-address, citable."""
|
| 365 |
+
rec, trace = _step(state, "ttm_311_forecast")
|
| 366 |
+
try:
|
| 367 |
+
if state.get("lat") is None:
|
| 368 |
+
rec["ok"] = False; rec["err"] = "no coords"
|
| 369 |
+
return state.update(ttm_311_forecast=None, trace=trace)
|
| 370 |
+
if not _in_nyc(state["lat"], state["lon"]):
|
| 371 |
+
rec["ok"] = False; rec["err"] = "out of NYC scope"
|
| 372 |
+
return state.update(ttm_311_forecast=None, trace=trace)
|
| 373 |
+
s = ttm_forecast.weekly_311_forecast_for_point(state["lat"], state["lon"])
|
| 374 |
+
rec["ok"] = bool(s.get("available"))
|
| 375 |
+
if not rec["ok"]:
|
| 376 |
+
rec["err"] = s.get("reason", "unavailable")
|
| 377 |
+
else:
|
| 378 |
+
rec["result"] = {
|
| 379 |
+
"history_total": s.get("history_total_complaints"),
|
| 380 |
+
"history_recent_mean": s.get("history_recent_3mo_mean"),
|
| 381 |
+
"forecast_mean": s.get("forecast_mean_per_week"),
|
| 382 |
+
"forecast_peak": s.get("forecast_peak_per_week"),
|
| 383 |
+
"accelerating": s.get("accelerating"),
|
| 384 |
+
}
|
| 385 |
+
return state.update(ttm_311_forecast=s, trace=trace)
|
| 386 |
+
except Exception as e:
|
| 387 |
+
rec["ok"] = False; rec["err"] = str(e)
|
| 388 |
+
log.exception("ttm_311_forecast failed")
|
| 389 |
+
return state.update(ttm_311_forecast=None, trace=trace)
|
| 390 |
+
finally:
|
| 391 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 392 |
+
|
| 393 |
+
|
| 394 |
+
@action(reads=["lat", "lon"], writes=["terramind", "trace"])
|
| 395 |
+
def step_terramind(state: State) -> State:
|
| 396 |
+
"""TerraMind v1 base β DEM β S2L2A synthesis as a per-query
|
| 397 |
+
cognitive-engine node. ~3-7s on M3 CPU. Output is a
|
| 398 |
+
*synthetic-prior* β explicitly fourth epistemic class alongside
|
| 399 |
+
empirical / modeled / proxy. Frame the doc body and reconciler
|
| 400 |
+
narration as 'plausible synthesis from terrain context', never
|
| 401 |
+
'imaged' or 'reconstructed'."""
|
| 402 |
+
rec, trace = _step(state, "terramind_synthesis")
|
| 403 |
+
try:
|
| 404 |
+
if state.get("lat") is None:
|
| 405 |
+
rec["ok"] = False; rec["err"] = "no coords"
|
| 406 |
+
return state.update(terramind=None, trace=trace)
|
| 407 |
+
if not _in_nyc(state["lat"], state["lon"]):
|
| 408 |
+
rec["ok"] = False; rec["err"] = "out of NYC scope"
|
| 409 |
+
return state.update(terramind=None, trace=trace)
|
| 410 |
+
from app.context import terramind_synthesis
|
| 411 |
+
s = terramind_synthesis.fetch(state["lat"], state["lon"])
|
| 412 |
+
rec["ok"] = bool(s.get("ok"))
|
| 413 |
+
if not s.get("ok"):
|
| 414 |
+
rec["err"] = s.get("err") or s.get("skipped") or "terramind unavailable"
|
| 415 |
+
else:
|
| 416 |
+
rec["result"] = {
|
| 417 |
+
"tim_chain": s.get("tim_chain"),
|
| 418 |
+
"diffusion_steps": s.get("diffusion_steps"),
|
| 419 |
+
"dem_mean_m": s.get("dem_mean_m"),
|
| 420 |
+
"synth_chip_shape": s.get("synth_chip_shape"),
|
| 421 |
+
"elapsed_s": s.get("elapsed_s"),
|
| 422 |
+
}
|
| 423 |
+
return state.update(terramind=s, trace=trace)
|
| 424 |
+
except Exception as e:
|
| 425 |
+
rec["ok"] = False; rec["err"] = str(e)
|
| 426 |
+
log.exception("terramind failed")
|
| 427 |
+
return state.update(terramind=None, trace=trace)
|
| 428 |
+
finally:
|
| 429 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 430 |
+
|
| 431 |
+
|
| 432 |
+
@action(reads=["lat", "lon"], writes=["noaa_tides", "trace"])
|
| 433 |
+
def step_noaa_tides(state: State) -> State:
|
| 434 |
+
rec, trace = _step(state, "noaa_tides")
|
| 435 |
+
try:
|
| 436 |
+
if state.get("lat") is None:
|
| 437 |
+
rec["ok"] = False; rec["err"] = "no coords"
|
| 438 |
+
return state.update(noaa_tides=None, trace=trace)
|
| 439 |
+
s = noaa_tides.summary_for_point(state["lat"], state["lon"])
|
| 440 |
+
rec["ok"] = s.get("error") is None
|
| 441 |
+
rec["result"] = {
|
| 442 |
+
"station": s["station_id"],
|
| 443 |
+
"observed_ft_mllw": s["observed_ft_mllw"],
|
| 444 |
+
"residual_ft": s["residual_ft"],
|
| 445 |
+
}
|
| 446 |
+
if s.get("error"): rec["err"] = s["error"]
|
| 447 |
+
return state.update(noaa_tides=s, trace=trace)
|
| 448 |
+
except Exception as e:
|
| 449 |
+
rec["ok"] = False; rec["err"] = str(e)
|
| 450 |
+
log.exception("noaa_tides failed")
|
| 451 |
+
return state.update(noaa_tides=None, trace=trace)
|
| 452 |
+
finally:
|
| 453 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 454 |
+
|
| 455 |
+
|
| 456 |
+
@action(reads=["lat", "lon"], writes=["nws_alerts", "trace"])
|
| 457 |
+
def step_nws_alerts(state: State) -> State:
|
| 458 |
+
rec, trace = _step(state, "nws_alerts")
|
| 459 |
+
try:
|
| 460 |
+
if state.get("lat") is None:
|
| 461 |
+
rec["ok"] = False; rec["err"] = "no coords"
|
| 462 |
+
return state.update(nws_alerts=None, trace=trace)
|
| 463 |
+
s = nws_alerts.summary_for_point(state["lat"], state["lon"])
|
| 464 |
+
rec["ok"] = s.get("error") is None
|
| 465 |
+
rec["result"] = {"n_active": s["n_active"]}
|
| 466 |
+
if s.get("error"): rec["err"] = s["error"]
|
| 467 |
+
return state.update(nws_alerts=s, trace=trace)
|
| 468 |
+
except Exception as e:
|
| 469 |
+
rec["ok"] = False; rec["err"] = str(e)
|
| 470 |
+
log.exception("nws_alerts failed")
|
| 471 |
+
return state.update(nws_alerts=None, trace=trace)
|
| 472 |
+
finally:
|
| 473 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 474 |
+
|
| 475 |
+
|
| 476 |
+
@action(reads=["lat", "lon"], writes=["nws_obs", "trace"])
|
| 477 |
+
def step_nws_obs(state: State) -> State:
|
| 478 |
+
rec, trace = _step(state, "nws_obs")
|
| 479 |
+
try:
|
| 480 |
+
if state.get("lat") is None:
|
| 481 |
+
rec["ok"] = False; rec["err"] = "no coords"
|
| 482 |
+
return state.update(nws_obs=None, trace=trace)
|
| 483 |
+
s = nws_obs.summary_for_point(state["lat"], state["lon"])
|
| 484 |
+
rec["ok"] = s.get("error") is None
|
| 485 |
+
rec["result"] = {
|
| 486 |
+
"station": s["station_id"],
|
| 487 |
+
"p1h_mm": s["precip_last_hour_mm"],
|
| 488 |
+
"p6h_mm": s["precip_last_6h_mm"],
|
| 489 |
+
}
|
| 490 |
+
if s.get("error"): rec["err"] = s["error"]
|
| 491 |
+
return state.update(nws_obs=s, trace=trace)
|
| 492 |
+
except Exception as e:
|
| 493 |
+
rec["ok"] = False; rec["err"] = str(e)
|
| 494 |
+
log.exception("nws_obs failed")
|
| 495 |
+
return state.update(nws_obs=None, trace=trace)
|
| 496 |
+
finally:
|
| 497 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 498 |
+
|
| 499 |
+
|
| 500 |
+
@action(reads=["lat", "lon"], writes=["ttm_forecast", "trace"])
|
| 501 |
+
def step_ttm_forecast(state: State) -> State:
|
| 502 |
+
"""Granite TTM r2 zero-shot forecast of the Battery surge residual."""
|
| 503 |
+
rec, trace = _step(state, "ttm_forecast")
|
| 504 |
+
try:
|
| 505 |
+
if state.get("lat") is None:
|
| 506 |
+
rec["ok"] = False; rec["err"] = "no coords"
|
| 507 |
+
return state.update(ttm_forecast=None, trace=trace)
|
| 508 |
+
s = ttm_forecast.summary_for_point(state["lat"], state["lon"])
|
| 509 |
+
if not s.get("available"):
|
| 510 |
+
rec["ok"] = False
|
| 511 |
+
rec["err"] = s.get("reason", "TTM unavailable")
|
| 512 |
+
return state.update(ttm_forecast=None, trace=trace)
|
| 513 |
+
rec["ok"] = True
|
| 514 |
+
rec["result"] = {
|
| 515 |
+
"context": s["context_length"],
|
| 516 |
+
"horizon": s["horizon_steps"],
|
| 517 |
+
"forecast_peak_ft": s["forecast_peak_ft"],
|
| 518 |
+
"forecast_peak_min_ahead": s["forecast_peak_minutes_ahead"],
|
| 519 |
+
"interesting": s["interesting"],
|
| 520 |
+
}
|
| 521 |
+
return state.update(ttm_forecast=s, trace=trace)
|
| 522 |
+
except Exception as e:
|
| 523 |
+
rec["ok"] = False; rec["err"] = str(e)
|
| 524 |
+
log.exception("ttm_forecast failed")
|
| 525 |
+
return state.update(ttm_forecast=None, trace=trace)
|
| 526 |
+
finally:
|
| 527 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 528 |
+
|
| 529 |
+
|
| 530 |
+
@action(reads=["lat", "lon"], writes=["ttm_battery_surge", "trace"])
|
| 531 |
+
def step_ttm_battery_surge(state: State) -> State:
|
| 532 |
+
"""Granite TTM r2 fine-tune β 96 h hourly Battery surge nowcast.
|
| 533 |
+
|
| 534 |
+
Same TTM r2 backbone family as step_ttm_forecast but a different
|
| 535 |
+
artefact: msradam/Granite-TTM-r2-Battery-Surge, trained on AMD
|
| 536 |
+
MI300X. Hourly cadence vs the zero-shot's 6-min, 4-day vs 9.6 h
|
| 537 |
+
horizon. Both can fire on the same query β the reconciler frames
|
| 538 |
+
each as a distinct forecast in the briefing."""
|
| 539 |
+
rec, trace = _step(state, "ttm_battery_surge")
|
| 540 |
+
try:
|
| 541 |
+
if state.get("lat") is None:
|
| 542 |
+
rec["ok"] = False; rec["err"] = "no coords"
|
| 543 |
+
return state.update(ttm_battery_surge=None, trace=trace)
|
| 544 |
+
# Battery gauge is a single point; the forecast applies citywide
|
| 545 |
+
# to NYC harbor entrance, so we don't gate by NYC bbox.
|
| 546 |
+
from app.live import ttm_battery_surge
|
| 547 |
+
s = ttm_battery_surge.fetch()
|
| 548 |
+
rec["ok"] = bool(s.get("available"))
|
| 549 |
+
if not rec["ok"]:
|
| 550 |
+
rec["err"] = s.get("reason", "unavailable")
|
| 551 |
+
return state.update(ttm_battery_surge=None, trace=trace)
|
| 552 |
+
rec["result"] = {
|
| 553 |
+
"context_h": s.get("context_hours"),
|
| 554 |
+
"horizon_h": s.get("horizon_hours"),
|
| 555 |
+
"forecast_peak_m": s.get("forecast_peak_m"),
|
| 556 |
+
"forecast_peak_hours_ahead": s.get("forecast_peak_hours_ahead"),
|
| 557 |
+
"interesting": s.get("interesting"),
|
| 558 |
+
}
|
| 559 |
+
return state.update(ttm_battery_surge=s, trace=trace)
|
| 560 |
+
except Exception as e:
|
| 561 |
+
rec["ok"] = False; rec["err"] = str(e)
|
| 562 |
+
log.exception("ttm_battery_surge failed")
|
| 563 |
+
return state.update(ttm_battery_surge=None, trace=trace)
|
| 564 |
+
finally:
|
| 565 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 566 |
+
|
| 567 |
+
|
| 568 |
+
@action(reads=["lat", "lon"], writes=["floodnet_forecast", "trace"])
|
| 569 |
+
def step_floodnet_forecast(state: State) -> State:
|
| 570 |
+
"""TTM r2 forecast of flood-event recurrence at the nearest FloodNet
|
| 571 |
+
sensor. Reuses the same (512, 96) singleton as ttm_311_forecast β no
|
| 572 |
+
additional model loaded into memory. Silent when the sensor has too
|
| 573 |
+
few historical events for a defensible forecast."""
|
| 574 |
+
rec, trace = _step(state, "floodnet_forecast")
|
| 575 |
+
try:
|
| 576 |
+
if state.get("lat") is None:
|
| 577 |
+
rec["ok"] = False; rec["err"] = "no coords"
|
| 578 |
+
return state.update(floodnet_forecast=None, trace=trace)
|
| 579 |
+
if not _in_nyc(state["lat"], state["lon"]):
|
| 580 |
+
rec["ok"] = False; rec["err"] = "out of NYC scope"
|
| 581 |
+
return state.update(floodnet_forecast=None, trace=trace)
|
| 582 |
+
s = fn_forecast.summary_for_point(state["lat"], state["lon"])
|
| 583 |
+
rec["ok"] = bool(s.get("available"))
|
| 584 |
+
if not rec["ok"]:
|
| 585 |
+
rec["err"] = s.get("reason", "unavailable")
|
| 586 |
+
else:
|
| 587 |
+
rec["result"] = {
|
| 588 |
+
"sensor_id": s.get("sensor_id"),
|
| 589 |
+
"distance_m": s.get("distance_from_query_m"),
|
| 590 |
+
"history_28d": s.get("history_recent_28d_events"),
|
| 591 |
+
"forecast_28d": s.get("forecast_28d_expected_events"),
|
| 592 |
+
"accelerating": s.get("accelerating"),
|
| 593 |
+
}
|
| 594 |
+
return state.update(floodnet_forecast=s if rec["ok"] else None,
|
| 595 |
+
trace=trace)
|
| 596 |
+
except Exception as e:
|
| 597 |
+
rec["ok"] = False; rec["err"] = str(e)
|
| 598 |
+
log.exception("floodnet_forecast failed")
|
| 599 |
+
return state.update(floodnet_forecast=None, trace=trace)
|
| 600 |
+
finally:
|
| 601 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 602 |
+
|
| 603 |
+
|
| 604 |
+
@action(reads=["lat", "lon"], writes=["npcc4_slr", "trace"])
|
| 605 |
+
def step_npcc4_projection(state: State) -> State:
|
| 606 |
+
"""NPCC4 (2024) sea-level rise table β static lookup, always available."""
|
| 607 |
+
rec, trace = _step(state, "npcc4_projection")
|
| 608 |
+
try:
|
| 609 |
+
s = npcc4_slr.get_projections()
|
| 610 |
+
rec["ok"] = True
|
| 611 |
+
rec["result"] = {
|
| 612 |
+
"2050_10th_in": s["2050"]["10"]["in"],
|
| 613 |
+
"2050_50th_in": s["2050"]["50"]["in"],
|
| 614 |
+
"2050_90th_in": s["2050"]["90"]["in"],
|
| 615 |
+
"2100_90th_in": s["2100"]["90"]["in"],
|
| 616 |
+
}
|
| 617 |
+
return state.update(npcc4_slr=s, trace=trace)
|
| 618 |
+
except Exception as e:
|
| 619 |
+
rec["ok"] = False; rec["err"] = str(e)
|
| 620 |
+
log.exception("npcc4_projection failed")
|
| 621 |
+
return state.update(npcc4_slr=None, trace=trace)
|
| 622 |
+
finally:
|
| 623 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 624 |
+
|
| 625 |
+
|
| 626 |
+
@action(reads=["lat", "lon"], writes=["mta_entrances", "trace"])
|
| 627 |
+
def step_mta_entrances(state: State) -> State:
|
| 628 |
+
rec, trace = _step(state, "mta_entrance_exposure")
|
| 629 |
+
try:
|
| 630 |
+
if state.get("lat") is None:
|
| 631 |
+
rec["ok"] = False; rec["err"] = "no coords"
|
| 632 |
+
return state.update(mta_entrances=None, trace=trace)
|
| 633 |
+
if not _in_nyc(state["lat"], state["lon"]):
|
| 634 |
+
rec["ok"] = False; rec["err"] = "out of NYC scope"
|
| 635 |
+
return state.update(mta_entrances=None, trace=trace)
|
| 636 |
+
s = r_mta.summary_for_point(state["lat"], state["lon"])
|
| 637 |
+
if not s.get("available"):
|
| 638 |
+
rec["ok"] = False; rec["err"] = "no entrances within radius"
|
| 639 |
+
return state.update(mta_entrances=None, trace=trace)
|
| 640 |
+
rec["ok"] = True
|
| 641 |
+
rec["result"] = {
|
| 642 |
+
"n_entrances": s["n_entrances"],
|
| 643 |
+
"n_inside_sandy_2012": s["n_inside_sandy_2012"],
|
| 644 |
+
"n_in_dep_extreme_2080": s["n_in_dep_extreme_2080"],
|
| 645 |
+
}
|
| 646 |
+
return state.update(mta_entrances=s, trace=trace)
|
| 647 |
+
except Exception as e:
|
| 648 |
+
rec["ok"] = False; rec["err"] = str(e)
|
| 649 |
+
log.exception("mta_entrances failed")
|
| 650 |
+
return state.update(mta_entrances=None, trace=trace)
|
| 651 |
+
finally:
|
| 652 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 653 |
+
|
| 654 |
+
|
| 655 |
+
@action(reads=["lat", "lon"], writes=["nycha_developments", "trace"])
|
| 656 |
+
def step_nycha(state: State) -> State:
|
| 657 |
+
rec, trace = _step(state, "nycha_development_exposure")
|
| 658 |
+
try:
|
| 659 |
+
if state.get("lat") is None:
|
| 660 |
+
rec["ok"] = False; rec["err"] = "no coords"
|
| 661 |
+
return state.update(nycha_developments=None, trace=trace)
|
| 662 |
+
if not _in_nyc(state["lat"], state["lon"]):
|
| 663 |
+
rec["ok"] = False; rec["err"] = "out of NYC scope"
|
| 664 |
+
return state.update(nycha_developments=None, trace=trace)
|
| 665 |
+
s = r_nycha.summary_for_point(state["lat"], state["lon"])
|
| 666 |
+
if not s.get("available"):
|
| 667 |
+
rec["ok"] = False; rec["err"] = "no NYCHA developments within radius"
|
| 668 |
+
return state.update(nycha_developments=None, trace=trace)
|
| 669 |
+
rec["ok"] = True
|
| 670 |
+
rec["result"] = {
|
| 671 |
+
"n_developments": s["n_developments"],
|
| 672 |
+
"n_inside_sandy_2012": s["n_inside_sandy_2012"],
|
| 673 |
+
"n_in_dep_extreme_2080": s["n_in_dep_extreme_2080"],
|
| 674 |
+
}
|
| 675 |
+
return state.update(nycha_developments=s, trace=trace)
|
| 676 |
+
except Exception as e:
|
| 677 |
+
rec["ok"] = False; rec["err"] = str(e)
|
| 678 |
+
log.exception("nycha failed")
|
| 679 |
+
return state.update(nycha_developments=None, trace=trace)
|
| 680 |
+
finally:
|
| 681 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 682 |
+
|
| 683 |
+
|
| 684 |
+
@action(reads=["lat", "lon"], writes=["doe_schools", "trace"])
|
| 685 |
+
def step_doe_schools(state: State) -> State:
|
| 686 |
+
rec, trace = _step(state, "doe_school_exposure")
|
| 687 |
+
try:
|
| 688 |
+
if state.get("lat") is None:
|
| 689 |
+
rec["ok"] = False; rec["err"] = "no coords"
|
| 690 |
+
return state.update(doe_schools=None, trace=trace)
|
| 691 |
+
if not _in_nyc(state["lat"], state["lon"]):
|
| 692 |
+
rec["ok"] = False; rec["err"] = "out of NYC scope"
|
| 693 |
+
return state.update(doe_schools=None, trace=trace)
|
| 694 |
+
s = r_schools.summary_for_point(state["lat"], state["lon"])
|
| 695 |
+
if not s.get("available"):
|
| 696 |
+
rec["ok"] = False; rec["err"] = "no schools within radius"
|
| 697 |
+
return state.update(doe_schools=None, trace=trace)
|
| 698 |
+
rec["ok"] = True
|
| 699 |
+
rec["result"] = {
|
| 700 |
+
"n_schools": s["n_schools"],
|
| 701 |
+
"n_inside_sandy_2012": s["n_inside_sandy_2012"],
|
| 702 |
+
"n_in_dep_extreme_2080": s["n_in_dep_extreme_2080"],
|
| 703 |
+
}
|
| 704 |
+
return state.update(doe_schools=s, trace=trace)
|
| 705 |
+
except Exception as e:
|
| 706 |
+
rec["ok"] = False; rec["err"] = str(e)
|
| 707 |
+
log.exception("doe_schools failed")
|
| 708 |
+
return state.update(doe_schools=None, trace=trace)
|
| 709 |
+
finally:
|
| 710 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 711 |
+
|
| 712 |
+
|
| 713 |
+
@action(reads=["lat", "lon"], writes=["doh_hospitals", "trace"])
|
| 714 |
+
def step_doh_hospitals(state: State) -> State:
|
| 715 |
+
rec, trace = _step(state, "doh_hospital_exposure")
|
| 716 |
+
try:
|
| 717 |
+
if state.get("lat") is None:
|
| 718 |
+
rec["ok"] = False; rec["err"] = "no coords"
|
| 719 |
+
return state.update(doh_hospitals=None, trace=trace)
|
| 720 |
+
if not _in_nyc(state["lat"], state["lon"]):
|
| 721 |
+
rec["ok"] = False; rec["err"] = "out of NYC scope"
|
| 722 |
+
return state.update(doh_hospitals=None, trace=trace)
|
| 723 |
+
s = r_hospitals.summary_for_point(state["lat"], state["lon"])
|
| 724 |
+
if not s.get("available"):
|
| 725 |
+
rec["ok"] = False; rec["err"] = "no hospitals within radius"
|
| 726 |
+
return state.update(doh_hospitals=None, trace=trace)
|
| 727 |
+
rec["ok"] = True
|
| 728 |
+
rec["result"] = {
|
| 729 |
+
"n_hospitals": s["n_hospitals"],
|
| 730 |
+
"n_inside_sandy_2012": s["n_inside_sandy_2012"],
|
| 731 |
+
"n_in_dep_extreme_2080": s["n_in_dep_extreme_2080"],
|
| 732 |
+
}
|
| 733 |
+
return state.update(doh_hospitals=s, trace=trace)
|
| 734 |
+
except Exception as e:
|
| 735 |
+
rec["ok"] = False; rec["err"] = str(e)
|
| 736 |
+
log.exception("doh_hospitals failed")
|
| 737 |
+
return state.update(doh_hospitals=None, trace=trace)
|
| 738 |
+
finally:
|
| 739 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 740 |
+
|
| 741 |
+
|
| 742 |
+
@action(reads=["lat", "lon"], writes=["microtopo", "trace"])
|
| 743 |
+
def step_microtopo(state: State) -> State:
|
| 744 |
+
rec, trace = _step(state, "microtopo_lidar")
|
| 745 |
+
try:
|
| 746 |
+
if state.get("lat") is None:
|
| 747 |
+
rec["ok"] = False; rec["err"] = "no coords"
|
| 748 |
+
return state.update(microtopo=None, trace=trace)
|
| 749 |
+
if not _in_nyc(state["lat"], state["lon"]):
|
| 750 |
+
rec["ok"] = False; rec["err"] = "out of NYC scope"
|
| 751 |
+
return state.update(microtopo=None, trace=trace)
|
| 752 |
+
m = microtopo.microtopo_at(state["lat"], state["lon"])
|
| 753 |
+
if m is None:
|
| 754 |
+
rec["ok"] = False; rec["err"] = "DEM fetch failed"
|
| 755 |
+
return state.update(microtopo=None, trace=trace)
|
| 756 |
+
rec["ok"] = True
|
| 757 |
+
rec["result"] = {
|
| 758 |
+
"elev_m": m.point_elev_m,
|
| 759 |
+
"pct_200m": m.rel_elev_pct_200m,
|
| 760 |
+
"relief_m": m.basin_relief_m,
|
| 761 |
+
}
|
| 762 |
+
return state.update(microtopo=vars(m), trace=trace)
|
| 763 |
+
except Exception as e:
|
| 764 |
+
rec["ok"] = False; rec["err"] = str(e)
|
| 765 |
+
log.exception("microtopo failed")
|
| 766 |
+
return state.update(microtopo=None, trace=trace)
|
| 767 |
+
finally:
|
| 768 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 769 |
+
|
| 770 |
+
|
| 771 |
+
|
| 772 |
+
|
| 773 |
+
@action(reads=["lat", "lon"], writes=["eo_chip", "trace"])
|
| 774 |
+
def step_eo_chip(state: State) -> State:
|
| 775 |
+
"""Fetch one S2L2A + S1RTC + DEM chip per query and stash it in
|
| 776 |
+
state for the TerraMind-NYC specialists.
|
| 777 |
+
|
| 778 |
+
Centralised so step_terramind_lulc and step_terramind_buildings
|
| 779 |
+
don't each re-fetch ~150 MB of imagery. Best-effort by design β
|
| 780 |
+
a deps-missing or no-scene outcome writes `{ok: False, skipped: ...}`
|
| 781 |
+
and the downstream TerraMind specialists silently no-op."""
|
| 782 |
+
rec, trace = _step(state, "eo_chip_fetch")
|
| 783 |
+
try:
|
| 784 |
+
if state.get("lat") is None:
|
| 785 |
+
rec["ok"] = False; rec["err"] = "no coords"
|
| 786 |
+
return state.update(eo_chip=None, trace=trace)
|
| 787 |
+
if not _in_nyc(state["lat"], state["lon"]):
|
| 788 |
+
rec["ok"] = False; rec["err"] = "out of NYC scope"
|
| 789 |
+
return state.update(eo_chip=None, trace=trace)
|
| 790 |
+
from app.context import eo_chip_cache
|
| 791 |
+
chip = eo_chip_cache.fetch(state["lat"], state["lon"])
|
| 792 |
+
rec["ok"] = bool(chip.get("ok"))
|
| 793 |
+
if not rec["ok"]:
|
| 794 |
+
rec["err"] = chip.get("skipped") or chip.get("err") or "unavailable"
|
| 795 |
+
else:
|
| 796 |
+
rec["result"] = {
|
| 797 |
+
"scene_id": (chip.get("s2_meta") or {}).get("scene_id"),
|
| 798 |
+
"scene_date": ((chip.get("s2_meta") or {}).get("datetime") or "")[:10],
|
| 799 |
+
"cloud_cover": (chip.get("s2_meta") or {}).get("cloud_cover"),
|
| 800 |
+
"has_s1": chip.get("s1") is not None,
|
| 801 |
+
"has_dem": chip.get("dem") is not None,
|
| 802 |
+
}
|
| 803 |
+
return state.update(eo_chip=chip, trace=trace)
|
| 804 |
+
except Exception as e:
|
| 805 |
+
rec["ok"] = False; rec["err"] = str(e)
|
| 806 |
+
log.exception("eo_chip failed")
|
| 807 |
+
return state.update(eo_chip=None, trace=trace)
|
| 808 |
+
finally:
|
| 809 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 810 |
+
|
| 811 |
+
|
| 812 |
+
@action(reads=["lat", "lon", "eo_chip"], writes=["terramind_lulc", "trace"])
|
| 813 |
+
def step_terramind_lulc(state: State) -> State:
|
| 814 |
+
"""5-class macro NYC LULC via msradam/TerraMind-NYC-Adapters.
|
| 815 |
+
|
| 816 |
+
Consumes the shared chip from step_eo_chip; if that didn't fire
|
| 817 |
+
cleanly this no-ops. Adapter loading (~1.6 GB base + ~325 MB LoRA)
|
| 818 |
+
is lazy on first call and cached across queries."""
|
| 819 |
+
rec, trace = _step(state, "terramind_lulc")
|
| 820 |
+
try:
|
| 821 |
+
if state.get("lat") is None:
|
| 822 |
+
rec["ok"] = False; rec["err"] = "no coords"
|
| 823 |
+
return state.update(terramind_lulc=None, trace=trace)
|
| 824 |
+
if not _in_nyc(state["lat"], state["lon"]):
|
| 825 |
+
rec["ok"] = False; rec["err"] = "out of NYC scope"
|
| 826 |
+
return state.update(terramind_lulc=None, trace=trace)
|
| 827 |
+
chip = state.get("eo_chip") or {}
|
| 828 |
+
if not chip.get("ok"):
|
| 829 |
+
rec["ok"] = False
|
| 830 |
+
rec["err"] = chip.get("skipped") or chip.get("err") or "no chip"
|
| 831 |
+
return state.update(terramind_lulc=None, trace=trace)
|
| 832 |
+
from app.context import terramind_nyc
|
| 833 |
+
tensors = chip.get("tensors") or {}
|
| 834 |
+
out = terramind_nyc.lulc(
|
| 835 |
+
tensors.get("S2L2A"),
|
| 836 |
+
s1rtc=tensors.get("S1RTC"),
|
| 837 |
+
dem=tensors.get("DEM"),
|
| 838 |
+
bounds_4326=chip.get("bounds_4326"),
|
| 839 |
+
)
|
| 840 |
+
rec["ok"] = bool(out.get("ok"))
|
| 841 |
+
if not rec["ok"]:
|
| 842 |
+
rec["err"] = out.get("skipped") or out.get("err") or "unavailable"
|
| 843 |
+
else:
|
| 844 |
+
rec["result"] = {
|
| 845 |
+
"dominant_class": out.get("dominant_class"),
|
| 846 |
+
"dominant_pct": out.get("dominant_pct"),
|
| 847 |
+
"n_classes_observed": len(out.get("class_fractions") or {}),
|
| 848 |
+
}
|
| 849 |
+
return state.update(terramind_lulc=out, trace=trace)
|
| 850 |
+
except Exception as e:
|
| 851 |
+
rec["ok"] = False; rec["err"] = str(e)
|
| 852 |
+
log.exception("terramind_lulc failed")
|
| 853 |
+
return state.update(terramind_lulc=None, trace=trace)
|
| 854 |
+
finally:
|
| 855 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 856 |
+
|
| 857 |
+
|
| 858 |
+
@action(reads=["lat", "lon", "eo_chip"],
|
| 859 |
+
writes=["terramind_buildings", "trace"])
|
| 860 |
+
def step_terramind_buildings(state: State) -> State:
|
| 861 |
+
"""Binary NYC building-footprint mask via msradam/TerraMind-NYC-Adapters."""
|
| 862 |
+
rec, trace = _step(state, "terramind_buildings")
|
| 863 |
+
try:
|
| 864 |
+
if state.get("lat") is None:
|
| 865 |
+
rec["ok"] = False; rec["err"] = "no coords"
|
| 866 |
+
return state.update(terramind_buildings=None, trace=trace)
|
| 867 |
+
if not _in_nyc(state["lat"], state["lon"]):
|
| 868 |
+
rec["ok"] = False; rec["err"] = "out of NYC scope"
|
| 869 |
+
return state.update(terramind_buildings=None, trace=trace)
|
| 870 |
+
chip = state.get("eo_chip") or {}
|
| 871 |
+
if not chip.get("ok"):
|
| 872 |
+
rec["ok"] = False
|
| 873 |
+
rec["err"] = chip.get("skipped") or chip.get("err") or "no chip"
|
| 874 |
+
return state.update(terramind_buildings=None, trace=trace)
|
| 875 |
+
from app.context import terramind_nyc
|
| 876 |
+
tensors = chip.get("tensors") or {}
|
| 877 |
+
out = terramind_nyc.buildings(
|
| 878 |
+
tensors.get("S2L2A"),
|
| 879 |
+
s1rtc=tensors.get("S1RTC"),
|
| 880 |
+
dem=tensors.get("DEM"),
|
| 881 |
+
bounds_4326=chip.get("bounds_4326"),
|
| 882 |
+
)
|
| 883 |
+
rec["ok"] = bool(out.get("ok"))
|
| 884 |
+
if not rec["ok"]:
|
| 885 |
+
rec["err"] = out.get("skipped") or out.get("err") or "unavailable"
|
| 886 |
+
else:
|
| 887 |
+
rec["result"] = {
|
| 888 |
+
"pct_buildings": out.get("pct_buildings"),
|
| 889 |
+
"n_building_components": out.get("n_building_components"),
|
| 890 |
+
}
|
| 891 |
+
return state.update(terramind_buildings=out, trace=trace)
|
| 892 |
+
except Exception as e:
|
| 893 |
+
rec["ok"] = False; rec["err"] = str(e)
|
| 894 |
+
log.exception("terramind_buildings failed")
|
| 895 |
+
return state.update(terramind_buildings=None, trace=trace)
|
| 896 |
+
finally:
|
| 897 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 898 |
+
|
| 899 |
+
|
| 900 |
+
@action(reads=["geocode", "sandy", "dep", "floodnet", "nyc311", "microtopo",
|
| 901 |
+
"ida_hwm", "prithvi_water", "noaa_tides", "nws_alerts", "nws_obs",
|
| 902 |
+
"ttm_forecast"],
|
| 903 |
+
writes=["rag", "trace"])
|
| 904 |
+
def step_rag(state: State) -> State:
|
| 905 |
+
rec, trace = _step(state, "rag_granite_embedding")
|
| 906 |
+
try:
|
| 907 |
+
geo = state.get("geocode") or {}
|
| 908 |
+
if not _in_nyc(geo.get("lat"), geo.get("lon")):
|
| 909 |
+
rec["ok"] = False; rec["err"] = "out of NYC scope"
|
| 910 |
+
return state.update(rag=[], trace=trace)
|
| 911 |
+
sandy = state.get("sandy")
|
| 912 |
+
dep = state.get("dep") or {}
|
| 913 |
+
# Build a context-rich query so retrieval pulls policy paragraphs
|
| 914 |
+
# relevant to *this* address, not generic flood text.
|
| 915 |
+
bits = []
|
| 916 |
+
if geo.get("address"):
|
| 917 |
+
bits.append(f"address {geo['address']}")
|
| 918 |
+
if geo.get("borough"):
|
| 919 |
+
bits.append(f"in {geo['borough']}")
|
| 920 |
+
if sandy:
|
| 921 |
+
bits.append("inside Hurricane Sandy 2012 inundation zone")
|
| 922 |
+
for v in dep.values():
|
| 923 |
+
if v.get("depth_class", 0) > 0:
|
| 924 |
+
bits.append(f"in {v['depth_label']} pluvial scenario")
|
| 925 |
+
bits.append("flood resilience plan, vulnerability, hardening, mitigation")
|
| 926 |
+
q = "; ".join(bits)
|
| 927 |
+
hits = rag_retrieve(q, k=3, min_score=0.45)
|
| 928 |
+
rec["ok"] = True
|
| 929 |
+
rec["result"] = {"hits": len(hits),
|
| 930 |
+
"top": [(h["doc_id"], round(h["score"], 2)) for h in hits]}
|
| 931 |
+
return state.update(rag=hits, trace=trace)
|
| 932 |
+
except Exception as e:
|
| 933 |
+
rec["ok"] = False; rec["err"] = str(e)
|
| 934 |
+
log.exception("rag failed")
|
| 935 |
+
return state.update(rag=[], trace=trace)
|
| 936 |
+
finally:
|
| 937 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 938 |
+
|
| 939 |
+
|
| 940 |
+
@action(reads=["rag"], writes=["gliner", "trace"])
|
| 941 |
+
def step_gliner(state: State) -> State:
|
| 942 |
+
"""GLiNER typed-entity extraction over the top RAG paragraphs.
|
| 943 |
+
|
| 944 |
+
Adds structured fields (`agency`, `dollar_amount`,
|
| 945 |
+
`infrastructure_project`, `nyc_location`, `date_range`) the
|
| 946 |
+
reconciler can cite with `[gliner_<source>]`. Silent no-op when
|
| 947 |
+
disabled via RIPRAP_GLINER_ENABLE=0 or when the model failed to
|
| 948 |
+
load β preserves the existing FSM contract.
|
| 949 |
+
"""
|
| 950 |
+
rec, trace = _step(state, "gliner_extract")
|
| 951 |
+
try:
|
| 952 |
+
from app.context.gliner_extract import extract_for_rag_hits
|
| 953 |
+
hits = state.get("rag") or []
|
| 954 |
+
if not hits:
|
| 955 |
+
rec["ok"] = True
|
| 956 |
+
rec["result"] = {"sources": 0, "skipped": "no rag hits"}
|
| 957 |
+
return state.update(gliner={}, trace=trace)
|
| 958 |
+
out = extract_for_rag_hits(hits)
|
| 959 |
+
rec["ok"] = True
|
| 960 |
+
rec["result"] = {
|
| 961 |
+
"sources": len(out),
|
| 962 |
+
"totals_by_label": _label_counts(out),
|
| 963 |
+
}
|
| 964 |
+
return state.update(gliner=out, trace=trace)
|
| 965 |
+
except Exception as e:
|
| 966 |
+
rec["ok"] = False
|
| 967 |
+
rec["err"] = str(e)
|
| 968 |
+
log.exception("gliner failed")
|
| 969 |
+
return state.update(gliner={}, trace=trace)
|
| 970 |
+
finally:
|
| 971 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 972 |
+
|
| 973 |
+
|
| 974 |
+
def _label_counts(gliner_out: dict[str, dict]) -> dict[str, int]:
|
| 975 |
+
counts: dict[str, int] = {}
|
| 976 |
+
for src in gliner_out.values():
|
| 977 |
+
for e in src.get("entities", []):
|
| 978 |
+
counts[e["label"]] = counts.get(e["label"], 0) + 1
|
| 979 |
+
return counts
|
| 980 |
+
|
| 981 |
+
|
| 982 |
+
@action(reads=["geocode", "sandy", "dep", "floodnet", "nyc311", "microtopo",
|
| 983 |
+
"ida_hwm", "prithvi_water", "prithvi_live", "terramind",
|
| 984 |
+
"terramind_lulc", "terramind_buildings",
|
| 985 |
+
"noaa_tides", "nws_alerts", "nws_obs", "ttm_forecast",
|
| 986 |
+
"ttm_311_forecast", "floodnet_forecast", "npcc4_slr",
|
| 987 |
+
"ttm_battery_surge",
|
| 988 |
+
"mta_entrances",
|
| 989 |
+
"nycha_developments", "doe_schools", "doh_hospitals",
|
| 990 |
+
"rag", "gliner"],
|
| 991 |
+
writes=["paragraph", "audit", "mellea", "citations", "trace"])
|
| 992 |
+
def step_reconcile(state: State) -> State:
|
| 993 |
+
is_strict = _current_strict_mode()
|
| 994 |
+
rec, trace = _step(state, "mellea_reconcile_address" if is_strict else "reconcile_granite41")
|
| 995 |
+
mellea_meta = None
|
| 996 |
+
try:
|
| 997 |
+
snap = {
|
| 998 |
+
"geocode": state.get("geocode"),
|
| 999 |
+
"sandy": state.get("sandy"),
|
| 1000 |
+
"dep": state.get("dep"),
|
| 1001 |
+
"floodnet": state.get("floodnet"),
|
| 1002 |
+
"nyc311": state.get("nyc311"),
|
| 1003 |
+
"microtopo": state.get("microtopo"),
|
| 1004 |
+
"ida_hwm": state.get("ida_hwm"),
|
| 1005 |
+
"prithvi_water": state.get("prithvi_water"),
|
| 1006 |
+
"noaa_tides": state.get("noaa_tides"),
|
| 1007 |
+
"nws_alerts": state.get("nws_alerts"),
|
| 1008 |
+
"nws_obs": state.get("nws_obs"),
|
| 1009 |
+
"ttm_forecast": state.get("ttm_forecast"),
|
| 1010 |
+
"ttm_311_forecast": state.get("ttm_311_forecast"),
|
| 1011 |
+
"floodnet_forecast": state.get("floodnet_forecast"),
|
| 1012 |
+
"npcc4_slr": state.get("npcc4_slr"),
|
| 1013 |
+
"ttm_battery_surge": state.get("ttm_battery_surge"),
|
| 1014 |
+
"rag": state.get("rag"),
|
| 1015 |
+
"gliner": state.get("gliner"),
|
| 1016 |
+
"prithvi_live": state.get("prithvi_live"),
|
| 1017 |
+
"terramind": state.get("terramind"),
|
| 1018 |
+
"terramind_lulc": state.get("terramind_lulc"),
|
| 1019 |
+
"terramind_buildings": state.get("terramind_buildings"),
|
| 1020 |
+
"mta_entrances": state.get("mta_entrances"),
|
| 1021 |
+
"nycha_developments": state.get("nycha_developments"),
|
| 1022 |
+
"doe_schools": state.get("doe_schools"),
|
| 1023 |
+
"doh_hospitals": state.get("doh_hospitals"),
|
| 1024 |
+
}
|
| 1025 |
+
if is_strict:
|
| 1026 |
+
from app.framing import augment_system_prompt
|
| 1027 |
+
from app.mellea_validator import DEFAULT_LOOP_BUDGET, reconcile_strict_streaming
|
| 1028 |
+
from app.reconcile import EXTRA_SYSTEM_PROMPT, build_documents, trim_docs_to_plan
|
| 1029 |
+
doc_msgs = build_documents(snap)
|
| 1030 |
+
doc_msgs = trim_docs_to_plan(doc_msgs, _current_planned_specialists())
|
| 1031 |
+
if not doc_msgs:
|
| 1032 |
+
para = "No grounded data available for this address."
|
| 1033 |
+
audit = {"raw": para, "dropped": []}
|
| 1034 |
+
else:
|
| 1035 |
+
token_cb = _current_token_callback()
|
| 1036 |
+
attempt_cb = _current_mellea_attempt_callback()
|
| 1037 |
+
framed_prompt = augment_system_prompt(
|
| 1038 |
+
EXTRA_SYSTEM_PROMPT,
|
| 1039 |
+
query=_current_user_query() or state.get("query") or "",
|
| 1040 |
+
intent=_current_planner_intent() or "single_address",
|
| 1041 |
+
)
|
| 1042 |
+
# Forward the (delta, attempt_idx) pair through. Older
|
| 1043 |
+
# token_cb signatures were single-arg; we detect by
|
| 1044 |
+
# introspecting the callable's expected positional count
|
| 1045 |
+
# so single_address.py's old shape still works while new
|
| 1046 |
+
# callbacks see the attempt index they need to clear the
|
| 1047 |
+
# frontend buffer on a Mellea reroll.
|
| 1048 |
+
def _fwd_token(delta: str, attempt_idx: int) -> None:
|
| 1049 |
+
if token_cb is None:
|
| 1050 |
+
return
|
| 1051 |
+
try:
|
| 1052 |
+
token_cb(delta, attempt_idx)
|
| 1053 |
+
except TypeError:
|
| 1054 |
+
token_cb(delta)
|
| 1055 |
+
mres = reconcile_strict_streaming(
|
| 1056 |
+
doc_msgs, framed_prompt,
|
| 1057 |
+
user_prompt="Write the cited paragraph now.",
|
| 1058 |
+
loop_budget=DEFAULT_LOOP_BUDGET,
|
| 1059 |
+
on_token=_fwd_token if token_cb else None,
|
| 1060 |
+
on_attempt_end=attempt_cb,
|
| 1061 |
+
)
|
| 1062 |
+
para = mres["paragraph"]
|
| 1063 |
+
audit = {"raw": para, "dropped": []}
|
| 1064 |
+
mellea_meta = {
|
| 1065 |
+
"rerolls": mres["rerolls"],
|
| 1066 |
+
"n_attempts": mres["n_attempts"],
|
| 1067 |
+
"requirements_passed": mres["requirements_passed"],
|
| 1068 |
+
"requirements_failed": mres["requirements_failed"],
|
| 1069 |
+
"requirements_total": mres["requirements_total"],
|
| 1070 |
+
"model": mres["model"],
|
| 1071 |
+
"loop_budget": mres["loop_budget"],
|
| 1072 |
+
}
|
| 1073 |
+
rec["result"] = {
|
| 1074 |
+
"rerolls": (mellea_meta or {}).get("rerolls"),
|
| 1075 |
+
"passed": (f"{len((mellea_meta or {}).get('requirements_passed') or [])}/"
|
| 1076 |
+
f"{(mellea_meta or {}).get('requirements_total') or 0}"),
|
| 1077 |
+
"paragraph_chars": len(para),
|
| 1078 |
+
}
|
| 1079 |
+
else:
|
| 1080 |
+
para, audit = run_reconcile(snap, return_audit=True,
|
| 1081 |
+
on_token=_current_token_callback())
|
| 1082 |
+
rec["result"] = {
|
| 1083 |
+
"paragraph_chars": len(para),
|
| 1084 |
+
"dropped_sentences": len(audit["dropped"]),
|
| 1085 |
+
}
|
| 1086 |
+
# Build citation metadata list from whichever doc_msgs were used.
|
| 1087 |
+
from app.reconcile import build_documents, trim_docs_to_plan
|
| 1088 |
+
_cite_msgs = build_documents(snap)
|
| 1089 |
+
_cite_msgs = trim_docs_to_plan(_cite_msgs, _current_planned_specialists())
|
| 1090 |
+
cite_list = citations_from_docs(_cite_msgs)
|
| 1091 |
+
rec["ok"] = True
|
| 1092 |
+
return state.update(paragraph=para, audit=audit,
|
| 1093 |
+
mellea=mellea_meta, citations=cite_list, trace=trace)
|
| 1094 |
+
except Exception as e:
|
| 1095 |
+
rec["ok"] = False; rec["err"] = str(e)
|
| 1096 |
+
log.exception("reconcile failed")
|
| 1097 |
+
return state.update(paragraph="", audit={"raw": "", "dropped": []},
|
| 1098 |
+
mellea=None, citations=[], trace=trace)
|
| 1099 |
+
finally:
|
| 1100 |
+
rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
|
| 1101 |
+
|
| 1102 |
+
|
| 1103 |
+
import os as _os # noqa: E402
|
| 1104 |
+
|
| 1105 |
+
|
| 1106 |
+
# Specialists that involve large spatial joins (every NYCHA development
|
| 1107 |
+
# overlapped against multiple flood layers, every DOE school footprint
|
| 1108 |
+
# joined to DEM/HAND, etc.) or per-query model inference (Prithvi-EO live
|
| 1109 |
+
# STAC + ViT, TerraMind diffusion). They're ~1-3 minutes apiece on a
|
| 1110 |
+
# laptop on the FIRST call (the lru_caches inside the registers warm up
|
| 1111 |
+
# afterwards). The previous parallel-fan-out FSM hid that cost behind
|
| 1112 |
+
# the longest single specialist; the linear FSM exposes it.
|
| 1113 |
+
#
|
| 1114 |
+
# Default OFF on local-Ollama so the demo briefing returns in well under
|
| 1115 |
+
# 90 s. Enable explicitly with RIPRAP_HEAVY_SPECIALISTS=1 (e.g. on the
|
| 1116 |
+
# AMD-vLLM path, where the reconciler's ~5 s leaves room for the joins).
|
| 1117 |
+
#
|
| 1118 |
+
# Remote ML lift: when RIPRAP_ML_BACKEND=remote (or auto with a base URL
|
| 1119 |
+
# set) the heavy specialists' GPU work runs on the droplet, so the local
|
| 1120 |
+
# wall-clock cost drops from ~60 s to ~5 s. Default ON in that case so
|
| 1121 |
+
# the public demo never silently disables them.
|
| 1122 |
+
def _remote_ml_configured() -> bool:
|
| 1123 |
+
backend = _os.environ.get("RIPRAP_ML_BACKEND", "auto").lower()
|
| 1124 |
+
if backend == "local":
|
| 1125 |
+
return False
|
| 1126 |
+
return bool(_os.environ.get("RIPRAP_ML_BASE_URL", "").strip())
|
| 1127 |
+
|
| 1128 |
+
|
| 1129 |
+
_HEAVY_DEFAULT = (
|
| 1130 |
+
"1" if (
|
| 1131 |
+
_os.environ.get("RIPRAP_LLM_PRIMARY", "ollama").lower() != "ollama"
|
| 1132 |
+
or _remote_ml_configured()
|
| 1133 |
+
) else "0"
|
| 1134 |
+
)
|
| 1135 |
+
_HEAVY_SPECIALISTS_ENABLED = _os.environ.get(
|
| 1136 |
+
"RIPRAP_HEAVY_SPECIALISTS", _HEAVY_DEFAULT,
|
| 1137 |
+
).lower() in ("1", "true", "yes")
|
| 1138 |
+
|
| 1139 |
+
# NYCHA / DOE / DOH registers load a 91 MB sandy_inundation.geojson via
|
| 1140 |
+
# geopandas on first call. On machines with slow I/O or single-threaded
|
| 1141 |
+
# Python GIL contention (M3 local dev) this takes 3β5 min and makes the
|
| 1142 |
+
# first single_address query appear hung. Disable by default; enable on
|
| 1143 |
+
# the AMD droplet where the server pre-warms these at startup.
|
| 1144 |
+
_NYCHA_REGISTERS_ENABLED = _os.environ.get(
|
| 1145 |
+
"RIPRAP_NYCHA_REGISTERS", "0",
|
| 1146 |
+
).lower() in ("1", "true", "yes")
|
| 1147 |
+
|
| 1148 |
+
|
| 1149 |
+
def build_app(query: str):
|
| 1150 |
+
"""Linear, single-action-per-step Burr application.
|
| 1151 |
+
|
| 1152 |
+
Order: cheap-first geo + flood layers, then live live network signals,
|
| 1153 |
+
then RAG β reconcile. Heavy specialists (NYCHA / DOE / DOH register
|
| 1154 |
+
joins, Prithvi-EO live STAC, TerraMind diffusion) are gated behind
|
| 1155 |
+
RIPRAP_HEAVY_SPECIALISTS β see the module-level note above.
|
| 1156 |
+
"""
|
| 1157 |
+
builder = (
|
| 1158 |
+
ApplicationBuilder()
|
| 1159 |
+
.with_state(query=query, trace=[])
|
| 1160 |
+
.with_entrypoint("geocode")
|
| 1161 |
+
)
|
| 1162 |
+
|
| 1163 |
+
actions: dict[str, Any] = {
|
| 1164 |
+
"geocode": step_geocode,
|
| 1165 |
+
"sandy": step_sandy,
|
| 1166 |
+
"dep": step_dep,
|
| 1167 |
+
"floodnet": step_floodnet,
|
| 1168 |
+
"nyc311": step_311,
|
| 1169 |
+
"noaa_tides": step_noaa_tides,
|
| 1170 |
+
"nws_alerts": step_nws_alerts,
|
| 1171 |
+
"nws_obs": step_nws_obs,
|
| 1172 |
+
"ttm_forecast": step_ttm_forecast,
|
| 1173 |
+
"ttm_311_forecast": step_ttm_311_forecast,
|
| 1174 |
+
"floodnet_forecast": step_floodnet_forecast,
|
| 1175 |
+
"npcc4_projection": step_npcc4_projection,
|
| 1176 |
+
"ttm_battery_surge": step_ttm_battery_surge,
|
| 1177 |
+
"microtopo": step_microtopo,
|
| 1178 |
+
"ida_hwm": step_ida_hwm,
|
| 1179 |
+
"mta_entrances": step_mta_entrances,
|
| 1180 |
+
"prithvi": step_prithvi, # baked GeoJSON polygons for Ida; cheap
|
| 1181 |
+
}
|
| 1182 |
+
if _HEAVY_SPECIALISTS_ENABLED and _NYCHA_REGISTERS_ENABLED:
|
| 1183 |
+
actions["nycha"] = step_nycha
|
| 1184 |
+
actions["doe_schools"] = step_doe_schools
|
| 1185 |
+
actions["doh_hospitals"] = step_doh_hospitals
|
| 1186 |
+
if _HEAVY_SPECIALISTS_ENABLED:
|
| 1187 |
+
actions["prithvi_live"] = step_prithvi_live
|
| 1188 |
+
actions["terramind"] = step_terramind
|
| 1189 |
+
# New TerraMind-NYC LoRA family β one chip fetch feeds two
|
| 1190 |
+
# specialists. Keep eo_chip directly before the two consumers
|
| 1191 |
+
# so the chip stays warm in memory and isn't garbage-collected
|
| 1192 |
+
# by anything in between.
|
| 1193 |
+
actions["eo_chip"] = step_eo_chip
|
| 1194 |
+
actions["terramind_lulc"] = step_terramind_lulc
|
| 1195 |
+
actions["terramind_buildings"] = step_terramind_buildings
|
| 1196 |
+
actions["rag"] = step_rag
|
| 1197 |
+
actions["gliner"] = step_gliner
|
| 1198 |
+
actions["reconcile"] = step_reconcile
|
| 1199 |
+
|
| 1200 |
+
# Sequential transitions β pair every adjacent action in the dict order.
|
| 1201 |
+
keys = list(actions.keys())
|
| 1202 |
+
transitions = list(zip(keys, keys[1:]))
|
| 1203 |
+
|
| 1204 |
+
return (
|
| 1205 |
+
builder.with_actions(**actions).with_transitions(*transitions).build()
|
| 1206 |
+
)
|
| 1207 |
+
|
| 1208 |
+
|
| 1209 |
+
def _summarize_energy(trace: list) -> dict | None:
|
| 1210 |
+
rec_step = next((t for t in trace if t.get("step") == "reconcile_granite41"
|
| 1211 |
+
and t.get("ok")), None)
|
| 1212 |
+
if not rec_step:
|
| 1213 |
+
return None
|
| 1214 |
+
total_s = sum(t.get("elapsed_s", 0) or 0 for t in trace)
|
| 1215 |
+
return energy_estimate(rec_step.get("elapsed_s", 0) or 0, total_s)
|
| 1216 |
+
|
| 1217 |
+
|
| 1218 |
+
def _summarize_emissions() -> dict | None:
|
| 1219 |
+
"""Snapshot the active per-call emissions tracker, if installed.
|
| 1220 |
+
|
| 1221 |
+
Returns None when no tracker is bound to this thread (e.g. unit
|
| 1222 |
+
tests that call `fsm.run` directly without going through the
|
| 1223 |
+
web/intent layer that installs one)."""
|
| 1224 |
+
t = emissions.current()
|
| 1225 |
+
return t.summarize() if t is not None else None
|
| 1226 |
+
|
| 1227 |
+
|
| 1228 |
+
def run(query: str) -> dict[str, Any]:
|
| 1229 |
+
app = build_app(query)
|
| 1230 |
+
final_action, _, final_state = app.run(halt_after=["reconcile"])
|
| 1231 |
+
trace = final_state.get("trace", [])
|
| 1232 |
+
return {
|
| 1233 |
+
"query": query,
|
| 1234 |
+
"geocode": final_state.get("geocode"),
|
| 1235 |
+
"sandy": final_state.get("sandy"),
|
| 1236 |
+
"dep": final_state.get("dep"),
|
| 1237 |
+
"floodnet": final_state.get("floodnet"),
|
| 1238 |
+
"nyc311": final_state.get("nyc311"),
|
| 1239 |
+
"microtopo": final_state.get("microtopo"),
|
| 1240 |
+
"ida_hwm": final_state.get("ida_hwm"),
|
| 1241 |
+
"prithvi_water": final_state.get("prithvi_water"),
|
| 1242 |
+
"terramind": final_state.get("terramind"),
|
| 1243 |
+
"terramind_lulc": final_state.get("terramind_lulc"),
|
| 1244 |
+
"terramind_buildings": final_state.get("terramind_buildings"),
|
| 1245 |
+
"eo_chip": final_state.get("eo_chip"),
|
| 1246 |
+
"noaa_tides": final_state.get("noaa_tides"),
|
| 1247 |
+
"nws_alerts": final_state.get("nws_alerts"),
|
| 1248 |
+
"nws_obs": final_state.get("nws_obs"),
|
| 1249 |
+
"ttm_forecast": final_state.get("ttm_forecast"),
|
| 1250 |
+
"ttm_311_forecast": final_state.get("ttm_311_forecast"),
|
| 1251 |
+
"floodnet_forecast": final_state.get("floodnet_forecast"),
|
| 1252 |
+
"ttm_battery_surge": final_state.get("ttm_battery_surge"),
|
| 1253 |
+
"mta_entrances": final_state.get("mta_entrances"),
|
| 1254 |
+
"nycha_developments": final_state.get("nycha_developments"),
|
| 1255 |
+
"doe_schools": final_state.get("doe_schools"),
|
| 1256 |
+
"doh_hospitals": final_state.get("doh_hospitals"),
|
| 1257 |
+
"rag": final_state.get("rag"),
|
| 1258 |
+
"paragraph": final_state.get("paragraph"),
|
| 1259 |
+
"audit": final_state.get("audit"),
|
| 1260 |
+
"mellea": final_state.get("mellea"),
|
| 1261 |
+
"energy": _summarize_energy(trace),
|
| 1262 |
+
"emissions": _summarize_emissions(),
|
| 1263 |
+
"trace": trace,
|
| 1264 |
+
}
|
| 1265 |
+
|
| 1266 |
+
|
| 1267 |
+
def iter_steps(query: str):
|
| 1268 |
+
"""Yield SSE-friendly events as the FSM runs.
|
| 1269 |
+
|
| 1270 |
+
Each Burr action emits exactly one trace record on completion; we
|
| 1271 |
+
yield it as a `step` event the moment the iterate loop returns from
|
| 1272 |
+
that action. Reconciler tokens stream through the threadlocal
|
| 1273 |
+
`set_token_callback` (installed before this generator is iterated),
|
| 1274 |
+
not through this queue.
|
| 1275 |
+
|
| 1276 |
+
Burr's `app.iterate(halt_after=["reconcile"])` runs synchronously,
|
| 1277 |
+
yielding `(action, result, state)` after every action. We drive it
|
| 1278 |
+
in a background thread so the per-action SSE events reach the
|
| 1279 |
+
client as soon as each action returns, while the reconciler's
|
| 1280 |
+
token callback fires concurrently from the same thread.
|
| 1281 |
+
"""
|
| 1282 |
+
import queue
|
| 1283 |
+
|
| 1284 |
+
q: queue.Queue[tuple[str, Any] | None] = queue.Queue()
|
| 1285 |
+
seen_keys: set[tuple[str, float]] = set()
|
| 1286 |
+
|
| 1287 |
+
def _push_step(rec: dict) -> None:
|
| 1288 |
+
key = (rec.get("step", ""), rec.get("started_at", 0.0))
|
| 1289 |
+
if key in seen_keys:
|
| 1290 |
+
return
|
| 1291 |
+
seen_keys.add(key)
|
| 1292 |
+
q.put(("step", rec))
|
| 1293 |
+
|
| 1294 |
+
app = build_app(query)
|
| 1295 |
+
final_state_holder: dict[str, Any] = {}
|
| 1296 |
+
|
| 1297 |
+
# Threadlocals are per-thread; the request thread (single_address.run
|
| 1298 |
+
# / neighborhood.run) sets the strict-mode flag, planner specialist
|
| 1299 |
+
# set, and token / Mellea-attempt callbacks, but Burr's app.iterate
|
| 1300 |
+
# runs in this generator's thread. Snapshot the request-thread state
|
| 1301 |
+
# and re-install on the iterate thread so step_reconcile sees them.
|
| 1302 |
+
_captured_strict = _current_strict_mode()
|
| 1303 |
+
_captured_planned = _current_planned_specialists()
|
| 1304 |
+
_captured_token_cb = _current_token_callback()
|
| 1305 |
+
_captured_mellea_cb = _current_mellea_attempt_callback()
|
| 1306 |
+
_captured_tracker = emissions.current()
|
| 1307 |
+
|
| 1308 |
+
def _run_iterate():
|
| 1309 |
+
set_strict_mode(_captured_strict)
|
| 1310 |
+
set_planned_specialists(_captured_planned)
|
| 1311 |
+
set_token_callback(_captured_token_cb)
|
| 1312 |
+
set_mellea_attempt_callback(_captured_mellea_cb)
|
| 1313 |
+
emissions.install(_captured_tracker)
|
| 1314 |
+
try:
|
| 1315 |
+
for _action_obj, _result, state in app.iterate(halt_after=["reconcile"]):
|
| 1316 |
+
final_state_holder["state"] = state
|
| 1317 |
+
# Each action appends one record to state.trace; emit the
|
| 1318 |
+
# most recent so the SSE client gets the step event the
|
| 1319 |
+
# moment Burr returns from that action.
|
| 1320 |
+
trace = state.get("trace") or []
|
| 1321 |
+
if trace:
|
| 1322 |
+
_push_step(trace[-1])
|
| 1323 |
+
except Exception as e:
|
| 1324 |
+
log.exception("iterate raised")
|
| 1325 |
+
q.put(("error", {"err": f"{type(e).__name__}: {e}"}))
|
| 1326 |
+
finally:
|
| 1327 |
+
set_strict_mode(False)
|
| 1328 |
+
set_planned_specialists(None)
|
| 1329 |
+
set_token_callback(None)
|
| 1330 |
+
set_mellea_attempt_callback(None)
|
| 1331 |
+
emissions.install(None)
|
| 1332 |
+
q.put(None) # sentinel
|
| 1333 |
+
|
| 1334 |
+
runner = _threading.Thread(target=_run_iterate, name="riprap-fsm",
|
| 1335 |
+
daemon=True)
|
| 1336 |
+
runner.start()
|
| 1337 |
+
|
| 1338 |
+
while True:
|
| 1339 |
+
item = q.get()
|
| 1340 |
+
if item is None:
|
| 1341 |
+
break
|
| 1342 |
+
kind, payload = item
|
| 1343 |
+
if kind == "step":
|
| 1344 |
+
yield {
|
| 1345 |
+
"kind": "step",
|
| 1346 |
+
"step": payload.get("step"),
|
| 1347 |
+
"ok": payload.get("ok"),
|
| 1348 |
+
"elapsed_s": payload.get("elapsed_s"),
|
| 1349 |
+
"result": payload.get("result"),
|
| 1350 |
+
"err": payload.get("err"),
|
| 1351 |
+
}
|
| 1352 |
+
elif kind == "error":
|
| 1353 |
+
yield {"kind": "error", **payload}
|
| 1354 |
+
|
| 1355 |
+
runner.join(timeout=5)
|
| 1356 |
+
state = final_state_holder.get("state")
|
| 1357 |
+
if state is None:
|
| 1358 |
+
yield {"kind": "final", "paragraph": "", "error": "FSM failed before any action completed"}
|
| 1359 |
+
return
|
| 1360 |
+
trace = state.get("trace", [])
|
| 1361 |
+
yield {
|
| 1362 |
+
"kind": "final",
|
| 1363 |
+
"geocode": state.get("geocode"),
|
| 1364 |
+
"sandy": state.get("sandy"),
|
| 1365 |
+
"dep": state.get("dep"),
|
| 1366 |
+
"floodnet": state.get("floodnet"),
|
| 1367 |
+
"nyc311": state.get("nyc311"),
|
| 1368 |
+
"microtopo": state.get("microtopo"),
|
| 1369 |
+
"ida_hwm": state.get("ida_hwm"),
|
| 1370 |
+
"prithvi_water": state.get("prithvi_water"),
|
| 1371 |
+
"prithvi_live": state.get("prithvi_live"),
|
| 1372 |
+
"terramind": state.get("terramind"),
|
| 1373 |
+
"terramind_lulc": state.get("terramind_lulc"),
|
| 1374 |
+
"terramind_buildings": state.get("terramind_buildings"),
|
| 1375 |
+
"noaa_tides": state.get("noaa_tides"),
|
| 1376 |
+
"nws_alerts": state.get("nws_alerts"),
|
| 1377 |
+
"nws_obs": state.get("nws_obs"),
|
| 1378 |
+
"ttm_forecast": state.get("ttm_forecast"),
|
| 1379 |
+
"ttm_311_forecast": state.get("ttm_311_forecast"),
|
| 1380 |
+
"floodnet_forecast": state.get("floodnet_forecast"),
|
| 1381 |
+
"ttm_battery_surge": state.get("ttm_battery_surge"),
|
| 1382 |
+
"mta_entrances": state.get("mta_entrances"),
|
| 1383 |
+
"nycha_developments": state.get("nycha_developments"),
|
| 1384 |
+
"doe_schools": state.get("doe_schools"),
|
| 1385 |
+
"doh_hospitals": state.get("doh_hospitals"),
|
| 1386 |
+
"rag": state.get("rag"),
|
| 1387 |
+
"gliner": state.get("gliner"),
|
| 1388 |
+
"paragraph": state.get("paragraph"),
|
| 1389 |
+
"audit": state.get("audit"),
|
| 1390 |
+
"mellea": state.get("mellea"),
|
| 1391 |
+
"citations": state.get("citations"),
|
| 1392 |
+
"energy": _summarize_energy(trace),
|
| 1393 |
+
"emissions": _summarize_emissions(),
|
| 1394 |
+
}
|
app/geocode.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Address geocoding β NYC primary + national fallback.
|
| 2 |
+
|
| 3 |
+
NYC primary: NYC DCP Geosearch (geosearch.planninglabs.nyc), no auth,
|
| 4 |
+
NYC-only. It will fuzzy-match upstate addresses to NYC streets β e.g.
|
| 5 |
+
'257 Washington Ave, Albany NY' silently maps to Clinton Hill, Brooklyn.
|
| 6 |
+
We detect this via a non-NYC region or non-NYC ZIP and fall back to
|
| 7 |
+
OpenStreetMap Nominatim (no key, free, rate-limited per usage policy).
|
| 8 |
+
|
| 9 |
+
Includes a borough-hint post-filter so Queens hyphenated-style addresses
|
| 10 |
+
(e.g. '153-09 90 Ave, Jamaica, Queens') preferentially resolve to the
|
| 11 |
+
borough the user named.
|
| 12 |
+
"""
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import logging
|
| 16 |
+
import re
|
| 17 |
+
from dataclasses import dataclass
|
| 18 |
+
|
| 19 |
+
import httpx
|
| 20 |
+
|
| 21 |
+
log = logging.getLogger("riprap.geocode")
|
| 22 |
+
|
| 23 |
+
URL = "https://geosearch.planninglabs.nyc/v2/search"
|
| 24 |
+
NOMINATIM_URL = "https://nominatim.openstreetmap.org/search"
|
| 25 |
+
NOMINATIM_UA = "Riprap-NYC/0.5 (civic-flood-tool; +https://huggingface.co/spaces/msradam/riprap-nyc)"
|
| 26 |
+
|
| 27 |
+
# NYC-bbox guard: lat 40.49β40.92, lon -74.27 to -73.69.
|
| 28 |
+
NYC_BBOX = (40.49, -74.27, 40.92, -73.69)
|
| 29 |
+
|
| 30 |
+
_UPSTATE_ZIP_RE = re.compile(r"\b1[2-4]\d{3}\b")
|
| 31 |
+
_BOROUGHS = ("Manhattan", "Bronx", "Brooklyn", "Queens", "Staten Island")
|
| 32 |
+
|
| 33 |
+
def _detect_borough(text: str) -> str | None:
|
| 34 |
+
t = text.lower()
|
| 35 |
+
for b in _BOROUGHS:
|
| 36 |
+
if b.lower() in t:
|
| 37 |
+
return b
|
| 38 |
+
# neighborhood -> borough hints
|
| 39 |
+
hints = {
|
| 40 |
+
"queens": "Queens", "jamaica": "Queens", "rockaway": "Queens",
|
| 41 |
+
"astoria": "Queens", "flushing": "Queens",
|
| 42 |
+
"manhattan": "Manhattan", "harlem": "Manhattan", "soho": "Manhattan",
|
| 43 |
+
"brooklyn": "Brooklyn", "bushwick": "Brooklyn", "red hook": "Brooklyn",
|
| 44 |
+
"bronx": "Bronx", "fordham": "Bronx",
|
| 45 |
+
"staten island": "Staten Island",
|
| 46 |
+
}
|
| 47 |
+
for needle, boro in hints.items():
|
| 48 |
+
if needle in t:
|
| 49 |
+
return boro
|
| 50 |
+
return None
|
| 51 |
+
|
| 52 |
+
@dataclass
|
| 53 |
+
class GeocodeHit:
|
| 54 |
+
address: str
|
| 55 |
+
borough: str | None
|
| 56 |
+
lat: float
|
| 57 |
+
lon: float
|
| 58 |
+
bbl: str | None
|
| 59 |
+
bin: str | None
|
| 60 |
+
raw: dict
|
| 61 |
+
|
| 62 |
+
def geocode(text: str, limit: int = 5) -> list[GeocodeHit]:
|
| 63 |
+
"""NYC Geosearch primary."""
|
| 64 |
+
try:
|
| 65 |
+
r = httpx.get(URL, params={"text": text, "size": limit}, timeout=5)
|
| 66 |
+
r.raise_for_status()
|
| 67 |
+
feats = r.json().get("features", [])
|
| 68 |
+
out = []
|
| 69 |
+
for f in feats:
|
| 70 |
+
p = f.get("properties", {})
|
| 71 |
+
coords = (f.get("geometry") or {}).get("coordinates") or [None, None]
|
| 72 |
+
out.append(GeocodeHit(
|
| 73 |
+
address=p.get("label") or p.get("name") or text,
|
| 74 |
+
borough=p.get("borough"),
|
| 75 |
+
lat=coords[1],
|
| 76 |
+
lon=coords[0],
|
| 77 |
+
bbl=p.get("addendum", {}).get("pad", {}).get("bbl"),
|
| 78 |
+
bin=p.get("addendum", {}).get("pad", {}).get("bin"),
|
| 79 |
+
raw=p,
|
| 80 |
+
))
|
| 81 |
+
return out
|
| 82 |
+
except Exception as e:
|
| 83 |
+
log.warning("Geosearch failed: %r", e)
|
| 84 |
+
return []
|
| 85 |
+
|
| 86 |
+
def geocode_nominatim(text: str) -> GeocodeHit | None:
|
| 87 |
+
"""National OSM Nominatim fallback."""
|
| 88 |
+
try:
|
| 89 |
+
r = httpx.get(NOMINATIM_URL, params={
|
| 90 |
+
"q": text, "format": "jsonv2", "addressdetails": "1",
|
| 91 |
+
"limit": 1, "countrycodes": "us",
|
| 92 |
+
}, headers={"User-Agent": NOMINATIM_UA}, timeout=10)
|
| 93 |
+
r.raise_for_status()
|
| 94 |
+
rows = r.json()
|
| 95 |
+
except Exception as e:
|
| 96 |
+
log.warning("Nominatim fetch failed: %r", e)
|
| 97 |
+
return None
|
| 98 |
+
if not rows:
|
| 99 |
+
return None
|
| 100 |
+
row = rows[0]
|
| 101 |
+
addr = row.get("address") or {}
|
| 102 |
+
|
| 103 |
+
# Try to map Nominatim borough/county back to NYC standard
|
| 104 |
+
boro = addr.get("suburb") or addr.get("city_district") or addr.get("county")
|
| 105 |
+
if boro and "Kings" in boro: boro = "Brooklyn"
|
| 106 |
+
if boro and "New York County" in boro: boro = "Manhattan"
|
| 107 |
+
if boro and "Queens" in boro: boro = "Queens"
|
| 108 |
+
if boro and "Bronx" in boro: boro = "Bronx"
|
| 109 |
+
if boro and "Richmond" in boro: boro = "Staten Island"
|
| 110 |
+
|
| 111 |
+
return GeocodeHit(
|
| 112 |
+
address=row.get("display_name") or text,
|
| 113 |
+
borough=boro,
|
| 114 |
+
lat=float(row["lat"]),
|
| 115 |
+
lon=float(row["lon"]),
|
| 116 |
+
bbl=None, # Nominatim doesn't have BBLs
|
| 117 |
+
bin=None,
|
| 118 |
+
raw={"source": "nominatim", **row},
|
| 119 |
+
)
|
| 120 |
+
|
| 121 |
+
def geocode_one(text: str) -> GeocodeHit | None:
|
| 122 |
+
"""Dynamic geocoder with failover."""
|
| 123 |
+
# 1. Try Geosearch
|
| 124 |
+
hits = geocode(text)
|
| 125 |
+
hint = _detect_borough(text)
|
| 126 |
+
|
| 127 |
+
if hint:
|
| 128 |
+
in_boro = [h for h in hits if h.borough and h.borough.lower() == hint.lower()]
|
| 129 |
+
if in_boro: return in_boro[0]
|
| 130 |
+
|
| 131 |
+
if hits:
|
| 132 |
+
top = hits[0]
|
| 133 |
+
if top.lat and 40.4 <= top.lat <= 41.0: # Broad NYC check
|
| 134 |
+
return top
|
| 135 |
+
|
| 136 |
+
# 2. Fall back to Nominatim
|
| 137 |
+
log.info("Falling back to Nominatim for %r", text)
|
| 138 |
+
return geocode_nominatim(text)
|
app/inference.py
ADDED
|
@@ -0,0 +1,268 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Remote-vs-local ML inference router.
|
| 2 |
+
|
| 3 |
+
Mirrors the call-surface shape of `app/llm.py` but for the non-LLM
|
| 4 |
+
heavy models (Prithvi, TerraMind, TTM, Granite Embedding, GLiNER).
|
| 5 |
+
|
| 6 |
+
The droplet runs a `riprap-models` FastAPI service alongside vLLM that
|
| 7 |
+
exposes an OpenAI-style endpoint per model class. When configured the
|
| 8 |
+
router POSTs the relevant payload there and returns the parsed response;
|
| 9 |
+
on connection error / 5xx / timeout it surfaces a typed exception that
|
| 10 |
+
caller modules catch and fall back to a local in-process model load.
|
| 11 |
+
|
| 12 |
+
Backend selection (env):
|
| 13 |
+
|
| 14 |
+
RIPRAP_ML_BACKEND = "remote" | "local" | "auto" (default: auto)
|
| 15 |
+
- remote: use only the droplet, raise if it errors
|
| 16 |
+
- local : never call the droplet, always use the
|
| 17 |
+
in-process model
|
| 18 |
+
- auto : try remote first, fall back to local if
|
| 19 |
+
remote is unreachable / errors out;
|
| 20 |
+
same semantics as app/llm.py
|
| 21 |
+
RIPRAP_ML_BASE_URL = http://129.212.181.238:8002 (no trailing slash)
|
| 22 |
+
RIPRAP_ML_API_KEY = <bearer token>
|
| 23 |
+
|
| 24 |
+
The router is *transport*-only β it does not own model bytes, weights,
|
| 25 |
+
or framework imports. Each specialist that wants remote inference calls
|
| 26 |
+
into the helpers below and provides its own local fallback. That keeps
|
| 27 |
+
the dependency graph clean: the local code path keeps working when the
|
| 28 |
+
RIPRAP_ML_* env is unset (e.g. on first-light dev or in unit tests).
|
| 29 |
+
"""
|
| 30 |
+
from __future__ import annotations
|
| 31 |
+
|
| 32 |
+
import base64
|
| 33 |
+
import logging
|
| 34 |
+
import os
|
| 35 |
+
import time
|
| 36 |
+
from collections.abc import Iterable
|
| 37 |
+
from typing import Any
|
| 38 |
+
|
| 39 |
+
import httpx
|
| 40 |
+
|
| 41 |
+
from app import emissions
|
| 42 |
+
|
| 43 |
+
log = logging.getLogger("riprap.inference")
|
| 44 |
+
|
| 45 |
+
_BACKEND = os.environ.get("RIPRAP_ML_BACKEND", "auto").lower()
|
| 46 |
+
_BASE_URL = os.environ.get("RIPRAP_ML_BASE_URL", "").rstrip("/")
|
| 47 |
+
_API_KEY = os.environ.get("RIPRAP_ML_API_KEY", "")
|
| 48 |
+
_DEFAULT_TIMEOUT = float(os.environ.get("RIPRAP_ML_TIMEOUT_S", "60"))
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
class RemoteUnreachable(RuntimeError):
|
| 52 |
+
"""Raised when the remote inference service is unconfigured, down,
|
| 53 |
+
times out, or returns 5xx. Callers catch this to fall through to a
|
| 54 |
+
local model load. 4xx errors propagate as the generic exception so
|
| 55 |
+
a caller bug doesn't get masked by a "fallback to local" path."""
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def remote_enabled() -> bool:
|
| 59 |
+
"""True iff the router is configured to attempt remote calls.
|
| 60 |
+
Returns False under explicit `local` mode or when the base URL is
|
| 61 |
+
empty (the auto-default with no env config)."""
|
| 62 |
+
if _BACKEND == "local":
|
| 63 |
+
return False
|
| 64 |
+
if not _BASE_URL:
|
| 65 |
+
return False
|
| 66 |
+
return True
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def _client(timeout: float | None = None) -> httpx.Client:
|
| 70 |
+
headers = {"User-Agent": "riprap-app/0.4.5"}
|
| 71 |
+
if _API_KEY:
|
| 72 |
+
headers["Authorization"] = f"Bearer {_API_KEY}"
|
| 73 |
+
return httpx.Client(
|
| 74 |
+
base_url=_BASE_URL,
|
| 75 |
+
headers=headers,
|
| 76 |
+
timeout=timeout if timeout is not None else _DEFAULT_TIMEOUT,
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def _post(path: str, payload: dict[str, Any], timeout: float | None = None) -> dict:
|
| 81 |
+
"""POST {payload} as JSON to the remote service's `path`. Returns the
|
| 82 |
+
parsed JSON body. Raises RemoteUnreachable on transport errors;
|
| 83 |
+
raises HTTPStatusError on 4xx so caller bugs surface."""
|
| 84 |
+
if not remote_enabled():
|
| 85 |
+
raise RemoteUnreachable("remote ML backend not configured "
|
| 86 |
+
"(RIPRAP_ML_BASE_URL empty or BACKEND=local)")
|
| 87 |
+
t0 = time.monotonic()
|
| 88 |
+
try:
|
| 89 |
+
with _client(timeout) as c:
|
| 90 |
+
r = c.post(path, json=payload)
|
| 91 |
+
except (httpx.ConnectError, httpx.ReadError, httpx.WriteError,
|
| 92 |
+
httpx.TimeoutException, httpx.RemoteProtocolError) as e:
|
| 93 |
+
raise RemoteUnreachable(f"{type(e).__name__}: {e}") from e
|
| 94 |
+
if r.status_code >= 500:
|
| 95 |
+
raise RemoteUnreachable(f"HTTP {r.status_code} from {path}: {r.text[:200]}")
|
| 96 |
+
r.raise_for_status()
|
| 97 |
+
duration_s = time.monotonic() - t0
|
| 98 |
+
# Hardware: msradam/riprap-vllm runs on NVIDIA L4. Operators can
|
| 99 |
+
# override via RIPRAP_HARDWARE_LABEL. The proxy reports per-call
|
| 100 |
+
# GPU energy off NVML in the X-GPU-Energy-J / X-GPU-Power-W headers
|
| 101 |
+
# β read those for a real measurement instead of the data-sheet
|
| 102 |
+
# estimate when present.
|
| 103 |
+
override = (os.environ.get("RIPRAP_HARDWARE_LABEL") or "").lower()
|
| 104 |
+
if "mi300x" in override or "amd" in override:
|
| 105 |
+
hw = "amd_mi300x"
|
| 106 |
+
elif "t4" in override:
|
| 107 |
+
hw = "nvidia_t4"
|
| 108 |
+
else:
|
| 109 |
+
hw = "nvidia_l4"
|
| 110 |
+
joules_real, power_w_real = _parse_gpu_headers(r.headers)
|
| 111 |
+
emissions.active().record_ml(
|
| 112 |
+
endpoint=path,
|
| 113 |
+
backend="riprap-models",
|
| 114 |
+
hardware=hw,
|
| 115 |
+
duration_s=duration_s,
|
| 116 |
+
joules_real=joules_real,
|
| 117 |
+
power_w_real=power_w_real,
|
| 118 |
+
)
|
| 119 |
+
return r.json()
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def _parse_gpu_headers(headers) -> tuple[float | None, float | None]:
|
| 123 |
+
"""Pull (joules, watts) from X-GPU-Energy-J / X-GPU-Power-W if the
|
| 124 |
+
proxy attached them. Returns (None, None) if the headers are absent
|
| 125 |
+
(older proxy build, NVML init failed, or the call streamed)."""
|
| 126 |
+
def _f(name: str) -> float | None:
|
| 127 |
+
v = headers.get(name)
|
| 128 |
+
if v is None or v == "":
|
| 129 |
+
return None
|
| 130 |
+
try:
|
| 131 |
+
return float(v)
|
| 132 |
+
except ValueError:
|
| 133 |
+
return None
|
| 134 |
+
return _f("x-gpu-energy-j"), _f("x-gpu-power-w")
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def _serialize_array(arr) -> str:
|
| 138 |
+
"""numpy/torch tensor β base64-encoded float32 raw bytes for transport.
|
| 139 |
+
Each remote handler decodes to (shape, dtype=float32) and reconstructs.
|
| 140 |
+
Reasonable round-trip for chips up to a few MB; large rasters should
|
| 141 |
+
use compressed numpy-savez instead β TODO when a model needs > 8 MB."""
|
| 142 |
+
import numpy as np
|
| 143 |
+
np_arr = arr if isinstance(arr, np.ndarray) else _to_numpy(arr)
|
| 144 |
+
np_arr = np_arr.astype("float32", copy=False)
|
| 145 |
+
return base64.b64encode(np_arr.tobytes()).decode("ascii")
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def _to_numpy(t):
|
| 149 |
+
"""Best-effort tensor β numpy. Accepts torch.Tensor or numpy already."""
|
| 150 |
+
try:
|
| 151 |
+
import torch
|
| 152 |
+
if isinstance(t, torch.Tensor):
|
| 153 |
+
return t.detach().cpu().numpy()
|
| 154 |
+
except ImportError:
|
| 155 |
+
pass
|
| 156 |
+
import numpy as np
|
| 157 |
+
return np.asarray(t)
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def _deserialize_array(b64: str, shape: list[int]):
|
| 161 |
+
"""Inverse of _serialize_array β bytes β numpy float32 with given shape."""
|
| 162 |
+
import numpy as np
|
| 163 |
+
raw = base64.b64decode(b64)
|
| 164 |
+
return np.frombuffer(raw, dtype="float32").reshape(shape)
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
# ---- Public router entry points -------------------------------------------
|
| 168 |
+
|
| 169 |
+
def healthcheck(timeout: float = 3.0) -> bool:
|
| 170 |
+
"""Quick reachability probe. True if the service responds 200 to GET
|
| 171 |
+
/healthz within `timeout` seconds. Used by /api/backend so the UI can
|
| 172 |
+
show whether the remote ML backend is currently live."""
|
| 173 |
+
if not remote_enabled():
|
| 174 |
+
return False
|
| 175 |
+
try:
|
| 176 |
+
with _client(timeout) as c:
|
| 177 |
+
r = c.get("/healthz")
|
| 178 |
+
return r.status_code == 200
|
| 179 |
+
except Exception:
|
| 180 |
+
return False
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def backend_info() -> dict[str, Any]:
|
| 184 |
+
"""Snapshot for /api/backend β what the UI should advertise."""
|
| 185 |
+
return {
|
| 186 |
+
"backend": _BACKEND,
|
| 187 |
+
"base_url": _BASE_URL or None,
|
| 188 |
+
"remote_enabled": remote_enabled(),
|
| 189 |
+
"reachable": healthcheck() if remote_enabled() else False,
|
| 190 |
+
}
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
def prithvi_pluvial(s2_chip, *, scene_id: str | None = None,
|
| 194 |
+
scene_datetime: str | None = None,
|
| 195 |
+
cloud_cover: float | None = None,
|
| 196 |
+
timeout: float | None = None) -> dict[str, Any]:
|
| 197 |
+
"""Remote forward pass through Prithvi-NYC-Pluvial v2.
|
| 198 |
+
Input: 6-band Sentinel-2 chip (numpy or torch, shape [6, H, W]).
|
| 199 |
+
Output: { ok, pct_water_within_500m, pct_water_full, scene_id, ... }.
|
| 200 |
+
Raises RemoteUnreachable if the service is down."""
|
| 201 |
+
arr = _to_numpy(s2_chip)
|
| 202 |
+
return _post("/v1/prithvi-pluvial", {
|
| 203 |
+
"s2": _serialize_array(arr),
|
| 204 |
+
"shape": list(arr.shape),
|
| 205 |
+
"scene_id": scene_id,
|
| 206 |
+
"scene_datetime": scene_datetime,
|
| 207 |
+
"cloud_cover": cloud_cover,
|
| 208 |
+
}, timeout=timeout)
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
def terramind(adapter: str, s2l2a=None, s1rtc=None, dem=None, *,
|
| 212 |
+
timeout: float | None = None) -> dict[str, Any]:
|
| 213 |
+
"""Remote forward through TerraMind-NYC-Adapters (LULC or Buildings)
|
| 214 |
+
or the v1 base generative path (synthesis). `adapter` is one of:
|
| 215 |
+
lulc, buildings, synthesis. Each modality is a numpy array, torch
|
| 216 |
+
tensor, or None β `synthesis` only needs DEM; the LoRA adapters
|
| 217 |
+
need at minimum S2L2A."""
|
| 218 |
+
payload: dict[str, Any] = {"adapter": adapter}
|
| 219 |
+
if s2l2a is not None:
|
| 220 |
+
s2_np = _to_numpy(s2l2a)
|
| 221 |
+
payload["s2"] = _serialize_array(s2_np)
|
| 222 |
+
payload["s2_shape"] = list(s2_np.shape)
|
| 223 |
+
if s1rtc is not None:
|
| 224 |
+
s1_np = _to_numpy(s1rtc)
|
| 225 |
+
payload["s1"] = _serialize_array(s1_np)
|
| 226 |
+
payload["s1_shape"] = list(s1_np.shape)
|
| 227 |
+
if dem is not None:
|
| 228 |
+
dem_np = _to_numpy(dem)
|
| 229 |
+
payload["dem"] = _serialize_array(dem_np)
|
| 230 |
+
payload["dem_shape"] = list(dem_np.shape)
|
| 231 |
+
return _post("/v1/terramind", payload, timeout=timeout)
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def ttm_forecast(model: str, history: Iterable[float], *,
|
| 235 |
+
context_length: int, prediction_length: int,
|
| 236 |
+
cadence: str = "h",
|
| 237 |
+
timeout: float | None = None) -> dict[str, Any]:
|
| 238 |
+
"""Remote Granite TTM r2 forecast.
|
| 239 |
+
`model` is one of: zero_shot_battery, fine_tune_battery, weekly_311,
|
| 240 |
+
floodnet_recurrence β the service decides which checkpoint to use.
|
| 241 |
+
`history` is a 1-D iterable of floats (the time series); `cadence`
|
| 242 |
+
is for the service's labelling (h / d / w / 6m). Output shape is
|
| 243 |
+
`{ ok, forecast: [...], peak_index, peak_value }`."""
|
| 244 |
+
series = list(map(float, history))
|
| 245 |
+
return _post("/v1/ttm-forecast", {
|
| 246 |
+
"model": model,
|
| 247 |
+
"history": series,
|
| 248 |
+
"context_length": context_length,
|
| 249 |
+
"prediction_length": prediction_length,
|
| 250 |
+
"cadence": cadence,
|
| 251 |
+
}, timeout=timeout)
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
def granite_embed(texts: list[str], *,
|
| 255 |
+
timeout: float | None = None) -> dict[str, Any]:
|
| 256 |
+
"""Remote Granite Embedding 278M batch encode.
|
| 257 |
+
Output: { ok, vectors: [[float, ...], ...] }. Vector dimension fixed
|
| 258 |
+
at 768 (granite-embedding-278m-multilingual)."""
|
| 259 |
+
return _post("/v1/granite-embed", {"texts": list(texts)}, timeout=timeout)
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
def gliner_extract(text: str, labels: list[str], *,
|
| 263 |
+
timeout: float | None = None) -> dict[str, Any]:
|
| 264 |
+
"""Remote GLiNER typed-entity extraction.
|
| 265 |
+
Output: { ok, entities: [{label, text, start, end, score}, ...] }."""
|
| 266 |
+
return _post("/v1/gliner-extract", {
|
| 267 |
+
"text": text, "labels": list(labels),
|
| 268 |
+
}, timeout=timeout)
|
app/intents/__init__.py
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Per-intent execution modules. Each intent knows how to take a planner
|
| 2 |
+
Plan and run only the specialists relevant to it, returning a
|
| 3 |
+
reconciler-ready set of documents and a paragraph."""
|