seriffic commited on
Commit
3dbff85
Β·
0 Parent(s):

deploy(l4): self-contained Riprap mirror

Browse files
This view is limited to 50 files because it contains too many changes. Β  See raw diff
Files changed (50) hide show
  1. .env.example +34 -0
  2. .gitattributes +54 -0
  3. .github/ISSUE_TEMPLATE/bug_report.yml +65 -0
  4. .github/ISSUE_TEMPLATE/config.yml +8 -0
  5. .github/ISSUE_TEMPLATE/feature_request.yml +54 -0
  6. .github/ISSUE_TEMPLATE/port_to_new_city.yml +70 -0
  7. .github/PULL_REQUEST_TEMPLATE.md +38 -0
  8. .github/workflows/check.yml +43 -0
  9. .gitignore +89 -0
  10. CHANGELOG.md +96 -0
  11. CODE_OF_CONDUCT.md +85 -0
  12. CONTRIBUTING.md +127 -0
  13. Dockerfile +104 -0
  14. README.md +26 -0
  15. SECURITY.md +54 -0
  16. agent.py +52 -0
  17. app/__init__.py +0 -0
  18. app/areas/__init__.py +0 -0
  19. app/areas/nta.py +224 -0
  20. app/assets/__init__.py +0 -0
  21. app/assets/mta_entrances.py +73 -0
  22. app/assets/nycha.py +28 -0
  23. app/assets/schools.py +27 -0
  24. app/context/__init__.py +0 -0
  25. app/context/_polygonize.py +165 -0
  26. app/context/dob_permits.py +258 -0
  27. app/context/eo_chip_cache.py +345 -0
  28. app/context/floodnet.py +148 -0
  29. app/context/gliner_extract.py +147 -0
  30. app/context/microtopo.py +274 -0
  31. app/context/noaa_tides.py +110 -0
  32. app/context/npcc4_slr.py +42 -0
  33. app/context/nws_alerts.py +71 -0
  34. app/context/nws_obs.py +108 -0
  35. app/context/nyc311.py +161 -0
  36. app/context/terramind_nyc.py +485 -0
  37. app/context/terramind_synthesis.py +468 -0
  38. app/emissions.py +269 -0
  39. app/energy.py +56 -0
  40. app/flood_layers/__init__.py +0 -0
  41. app/flood_layers/dep_stormwater.py +168 -0
  42. app/flood_layers/ida_hwm.py +96 -0
  43. app/flood_layers/prithvi_live.py +563 -0
  44. app/flood_layers/prithvi_water.py +120 -0
  45. app/flood_layers/sandy_inundation.py +110 -0
  46. app/framing.py +249 -0
  47. app/fsm.py +1394 -0
  48. app/geocode.py +138 -0
  49. app/inference.py +268 -0
  50. app/intents/__init__.py +3 -0
.env.example ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Riprap environment configuration.
2
+ #
3
+ # Copy this file to `.env` and fill in the values that match the
4
+ # inference backend you want to talk to. The default profile runs
5
+ # only the app container, so both the LLM (vLLM serving Granite 4.1)
6
+ # and the ML specialist service must be reachable at HTTP endpoints.
7
+ #
8
+ # Three common configurations:
9
+ #
10
+ # 1. Easiest β€” talk to the live demo's backends. Adam runs a public
11
+ # MI300X droplet for the hackathon; if it's still up at demo time,
12
+ # both endpoints are reachable from anywhere.
13
+ #
14
+ # 2. Self-hosted β€” bring up your own MI300X droplet via
15
+ # docs/DROPLET-RUNBOOK.md, then point both URLs at it.
16
+ #
17
+ # 3. Full local β€” use `docker compose --profile with-models up` to
18
+ # run the riprap-models service yourself (requires a GPU on your
19
+ # box) and point a separate vLLM container at Granite 4.1.
20
+
21
+ # ---- Granite 4.1 reconciler (vLLM, OpenAI-compatible) -----------------
22
+ # Set to "ollama" instead of "vllm" if you have a local Ollama with
23
+ # granite4.1:8b pulled and want to use that.
24
+ RIPRAP_LLM_PRIMARY=vllm
25
+ RIPRAP_LLM_BASE_URL=http://your-vllm-host:8000/v1
26
+ RIPRAP_LLM_API_KEY=your-token-here
27
+
28
+ # ---- ML specialist service (Prithvi, TerraMind, GLiNER, etc.) ---------
29
+ RIPRAP_ML_BASE_URL=http://your-ml-host:7860
30
+ RIPRAP_ML_API_KEY=your-token-here
31
+
32
+ # ---- Backend pill labels (cosmetic, shown top-right of the UI) --------
33
+ RIPRAP_HARDWARE_LABEL=AMD MI300X
34
+ RIPRAP_ENGINE_LABEL=Granite 4.1 / vLLM
.gitattributes ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Riprap-specific LFS tracking
2
+ *.geojson filter=lfs diff=lfs merge=lfs -text
3
+ *.tif filter=lfs diff=lfs merge=lfs -text
4
+ *.pdf filter=lfs diff=lfs merge=lfs -text
5
+ # Pre-computed register paragraphs
6
+ data/registers/*.json filter=lfs diff=lfs merge=lfs -text
7
+ # Esri FileGDB internal binary files (DEP Stormwater scenario data)
8
+ *.gdbtable filter=lfs diff=lfs merge=lfs -text
9
+ *.gdbtablx filter=lfs diff=lfs merge=lfs -text
10
+ *.gdbindexes filter=lfs diff=lfs merge=lfs -text
11
+ *.atx filter=lfs diff=lfs merge=lfs -text
12
+ *.spx filter=lfs diff=lfs merge=lfs -text
13
+ *.freelist filter=lfs diff=lfs merge=lfs -text
14
+ *.horizon filter=lfs diff=lfs merge=lfs -text
15
+ *.FDO_UUID filter=lfs diff=lfs merge=lfs -text
16
+ # Hugging Face's standard LFS rules (kept for forward-compat with model assets)
17
+ *.7z filter=lfs diff=lfs merge=lfs -text
18
+ *.arrow filter=lfs diff=lfs merge=lfs -text
19
+ *.bin filter=lfs diff=lfs merge=lfs -text
20
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
21
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
22
+ *.ftz filter=lfs diff=lfs merge=lfs -text
23
+ *.gz filter=lfs diff=lfs merge=lfs -text
24
+ *.h5 filter=lfs diff=lfs merge=lfs -text
25
+ *.joblib filter=lfs diff=lfs merge=lfs -text
26
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
27
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
28
+ *.model filter=lfs diff=lfs merge=lfs -text
29
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
30
+ *.npy filter=lfs diff=lfs merge=lfs -text
31
+ *.npz filter=lfs diff=lfs merge=lfs -text
32
+ *.onnx filter=lfs diff=lfs merge=lfs -text
33
+ *.ot filter=lfs diff=lfs merge=lfs -text
34
+ *.parquet filter=lfs diff=lfs merge=lfs -text
35
+ *.pb filter=lfs diff=lfs merge=lfs -text
36
+ *.pickle filter=lfs diff=lfs merge=lfs -text
37
+ *.pkl filter=lfs diff=lfs merge=lfs -text
38
+ *.pt filter=lfs diff=lfs merge=lfs -text
39
+ *.pth filter=lfs diff=lfs merge=lfs -text
40
+ *.rar filter=lfs diff=lfs merge=lfs -text
41
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
42
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
43
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
44
+ *.tar filter=lfs diff=lfs merge=lfs -text
45
+ *.tflite filter=lfs diff=lfs merge=lfs -text
46
+ *.tgz filter=lfs diff=lfs merge=lfs -text
47
+ *.wasm filter=lfs diff=lfs merge=lfs -text
48
+ *.xz filter=lfs diff=lfs merge=lfs -text
49
+ *.zip filter=lfs diff=lfs merge=lfs -text
50
+ *.zst filter=lfs diff=lfs merge=lfs -text
51
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
52
+ *.pptx filter=lfs diff=lfs merge=lfs -text
53
+ assets/screenshots/** filter=lfs diff=lfs merge=lfs -text
54
+ slides/*.png filter=lfs diff=lfs merge=lfs -text
.github/ISSUE_TEMPLATE/bug_report.yml ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: Bug report
2
+ description: A briefing came back wrong, a Stone failed to fire, or the UI broke.
3
+ title: "[bug] "
4
+ labels: ["bug"]
5
+ body:
6
+ - type: markdown
7
+ attributes:
8
+ value: |
9
+ Thanks for filing! Riprap is a hackathon-period demo; the more
10
+ reproducible the report, the faster it gets fixed.
11
+ - type: input
12
+ id: address
13
+ attributes:
14
+ label: NYC address tested
15
+ description: The exact string you typed (or "n/a" if the bug is UI-only).
16
+ placeholder: 80 Pioneer Street, Brooklyn
17
+ validations:
18
+ required: true
19
+ - type: textarea
20
+ id: expected
21
+ attributes:
22
+ label: Expected behavior
23
+ validations:
24
+ required: true
25
+ - type: textarea
26
+ id: actual
27
+ attributes:
28
+ label: Actual behavior
29
+ description: Paste the briefing text or describe the failure.
30
+ validations:
31
+ required: true
32
+ - type: dropdown
33
+ id: surface
34
+ attributes:
35
+ label: Where did you reproduce this?
36
+ options:
37
+ - Hosted demo (lablab Space)
38
+ - Local Docker (`docker compose up`)
39
+ - Local dev server (`uvicorn web.main:app`)
40
+ - Self-hosted GPU inference
41
+ validations:
42
+ required: true
43
+ - type: input
44
+ id: browser
45
+ attributes:
46
+ label: Browser / OS
47
+ placeholder: Chrome 142 on macOS 14
48
+ - type: textarea
49
+ id: console
50
+ attributes:
51
+ label: Browser console errors
52
+ description: DevTools β†’ Console. Paste anything red.
53
+ render: text
54
+ - type: textarea
55
+ id: stream
56
+ attributes:
57
+ label: /api/agent/stream output (optional)
58
+ description: |
59
+ If the bug is a Stone failure, paste the relevant lines from the
60
+ SSE trace pane (or curl `/api/agent/stream?q=<address>` directly).
61
+ render: text
62
+ - type: textarea
63
+ id: notes
64
+ attributes:
65
+ label: Anything else
.github/ISSUE_TEMPLATE/config.yml ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ blank_issues_enabled: false
2
+ contact_links:
3
+ - name: Try the live demo
4
+ url: https://lablab-ai-amd-developer-hackathon-riprap-nyc.hf.space
5
+ about: Reproduce the issue against the hosted Space before filing.
6
+ - name: Read the architecture docs
7
+ url: https://github.com/msradam/riprap-nyc/tree/main/docs
8
+ about: ARCHITECTURE, METHODOLOGY, EMISSIONS, DEPLOY, BENCHMARKS, RESEARCH.
.github/ISSUE_TEMPLATE/feature_request.yml ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: Feature request
2
+ description: Propose a new probe, a new Stone, or a new civic-tech use case.
3
+ title: "[feat] "
4
+ labels: ["enhancement"]
5
+ body:
6
+ - type: textarea
7
+ id: usecase
8
+ attributes:
9
+ label: Civic-tech use case
10
+ description: |
11
+ Who is the user, what decision are they making, and what
12
+ evidence would Riprap need to surface to support it?
13
+ placeholder: |
14
+ e.g. "A resilience office siting a capital project needs the
15
+ joint exposure of NYCHA + schools within 200m of a Sandy
16
+ 100-year inundation polygon."
17
+ validations:
18
+ required: true
19
+ - type: textarea
20
+ id: data
21
+ attributes:
22
+ label: Data source(s)
23
+ description: |
24
+ Which public-record datasets should Riprap pull from? Include
25
+ URLs, agency owner, refresh cadence, and licence if known.
26
+ validations:
27
+ required: true
28
+ - type: dropdown
29
+ id: stone
30
+ attributes:
31
+ label: Which Stone does this belong in?
32
+ options:
33
+ - Cornerstone (hazard memory)
34
+ - Keystone (asset registers)
35
+ - Touchstone (live observation)
36
+ - Lodestone (forecast)
37
+ - Capstone (synthesis)
38
+ - Not sure / cross-cutting
39
+ validations:
40
+ required: true
41
+ - type: dropdown
42
+ id: contribute
43
+ attributes:
44
+ label: Willing to contribute the implementation?
45
+ options:
46
+ - "Yes β€” I can open the PR"
47
+ - "Maybe β€” with mentorship"
48
+ - "No β€” flagging the gap"
49
+ validations:
50
+ required: true
51
+ - type: textarea
52
+ id: notes
53
+ attributes:
54
+ label: Anything else
.github/ISSUE_TEMPLATE/port_to_new_city.yml ADDED
@@ -0,0 +1,70 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: Port to a new city
2
+ description: Plan a Riprap deployment for a city other than NYC.
3
+ title: "[port] "
4
+ labels: ["port", "enhancement"]
5
+ body:
6
+ - type: markdown
7
+ attributes:
8
+ value: |
9
+ Riprap's Five Stones taxonomy is city-agnostic; only the probes
10
+ plugged into each Stone change. See the "Five Stones beyond NYC"
11
+ section in the README. This template helps scope a port.
12
+ - type: input
13
+ id: city
14
+ attributes:
15
+ label: Target city / region
16
+ placeholder: e.g. Houston, TX
17
+ validations:
18
+ required: true
19
+ - type: textarea
20
+ id: cornerstone
21
+ attributes:
22
+ label: Cornerstone β€” hazard memory
23
+ description: |
24
+ Local historical inundation extents, regional DEM, regulatory
25
+ floodplain maps. Include dataset URLs and licences.
26
+ validations:
27
+ required: true
28
+ - type: textarea
29
+ id: keystone
30
+ attributes:
31
+ label: Keystone β€” asset registers
32
+ description: |
33
+ Transit, housing, education, healthcare polygons your jurisdiction
34
+ publishes.
35
+ validations:
36
+ required: true
37
+ - type: textarea
38
+ id: touchstone
39
+ attributes:
40
+ label: Touchstone β€” live observation
41
+ description: |
42
+ Live sensors, complaint streams (e.g. Houston has FloodNet
43
+ analogues; many cities expose 311 or equivalent).
44
+ validations:
45
+ required: true
46
+ - type: textarea
47
+ id: lodestone
48
+ attributes:
49
+ label: Lodestone β€” forecast
50
+ description: |
51
+ Local NWS / hydrologic / surge models, tide gauges, time-series
52
+ fine-tunes you'd retrain.
53
+ validations:
54
+ required: true
55
+ - type: dropdown
56
+ id: hardware
57
+ attributes:
58
+ label: Target inference hardware
59
+ options:
60
+ - AMD MI300X (or other ROCm)
61
+ - NVIDIA L4 / A10
62
+ - NVIDIA H100 / A100
63
+ - CPU-only (Ollama)
64
+ - Not decided
65
+ validations:
66
+ required: true
67
+ - type: textarea
68
+ id: notes
69
+ attributes:
70
+ label: Anything else
.github/PULL_REQUEST_TEMPLATE.md ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!-- Thanks for opening a PR. The checklist below mirrors how Riprap
2
+ was kept stable through hackathon week. -->
3
+
4
+ ## Summary
5
+
6
+ <!-- One paragraph: what changed, and why. Reference issues with #N. -->
7
+
8
+ ## Tested against
9
+
10
+ - [ ] Local dev server (`uvicorn web.main:app`)
11
+ - [ ] Local Docker (`docker compose up`)
12
+ - [ ] Hosted lablab Space
13
+ - [ ] Self-hosted GPU inference
14
+
15
+ ## Stones-fire probe
16
+
17
+ <!-- Paste the tail of `scripts/probe_stones_fire.py` output. The PR
18
+ should not be merged unless all five Stones fire. -->
19
+
20
+ ```
21
+ PYTHONPATH=. uv run python scripts/probe_stones_fire.py --timeout 600
22
+ ```
23
+
24
+ ## Energy-ledger sanity check
25
+
26
+ <!-- If this PR touches inference, app/emissions.py, or app/llm.py:
27
+ paste the n_measured / n_calls ratio and confirm hardware label. -->
28
+
29
+ ## Checklist
30
+
31
+ - [ ] No regression in `app/`, `web/`, `services/`, or
32
+ `inference-vllm/proxy.py` logic (typo-only edits OK).
33
+ - [ ] Docs updated (`README.md`, relevant `docs/*.md`) if public
34
+ surface changed.
35
+ - [ ] `CHANGELOG.md` entry under `[Unreleased]` with the right
36
+ `Added` / `Changed` / `Fixed` bucket.
37
+ - [ ] Conventional-commit prefix on the squash title
38
+ (`feat:` / `fix:` / `docs:` / `chore:` / `build:`).
.github/workflows/check.yml ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: check
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+ branches: [main]
8
+
9
+ permissions:
10
+ contents: read
11
+
12
+ jobs:
13
+ check:
14
+ name: import + lightweight tests
15
+ runs-on: ubuntu-latest
16
+ timeout-minutes: 15
17
+ steps:
18
+ - uses: actions/checkout@v4
19
+
20
+ - name: Install uv
21
+ uses: astral-sh/setup-uv@v3
22
+ with:
23
+ enable-cache: true
24
+
25
+ - name: Set up Python 3.12
26
+ run: uv python install 3.12
27
+
28
+ - name: Create venv and install deps
29
+ run: |
30
+ uv venv --python 3.12
31
+ uv pip install -r requirements.txt
32
+
33
+ - name: Import smoke test
34
+ env:
35
+ PYTHONPATH: .
36
+ run: |
37
+ uv run python -c "from app import fsm, llm, inference, emissions; from web import main"
38
+
39
+ - name: Lightweight pytest subset
40
+ env:
41
+ PYTHONPATH: .
42
+ run: |
43
+ uv run pytest -q tests/test_stones.py tests/test_compare_shape.py tests/test_stone_envelope.py
.gitignore ADDED
@@ -0,0 +1,89 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Session artifacts (legacy agent reports β€” not for the public repo)
2
+ *MORNING-BRIEF*.md
3
+ *OVERNIGHT*.md
4
+ *COMMS-OVERNIGHT*.md
5
+ CODE-MORNING-BRIEF*.md
6
+ MONDAY.md
7
+ FRIDAY*.md
8
+ *-REPORT.md
9
+ docs/sessions/
10
+ docs/design_handoff/
11
+
12
+ # Local-only secrets / credentials
13
+ AMD_TOKEN
14
+
15
+ # Probe / batch / diagnostic output (regenerable; not for the repo)
16
+ tests/batch_results.json
17
+ tests/overnight_audit.json
18
+ scripts/diagnostic_*.py
19
+ scripts/find_top_locations.py
20
+ scripts/verify_locations.py
21
+
22
+ __pycache__/
23
+ *.py[cod]
24
+ *.egg-info/
25
+ dist/
26
+ build/
27
+ .venv/
28
+ .env
29
+ .DS_Store
30
+ outputs/
31
+ node_modules/
32
+ *.tmp
33
+ *.log
34
+ .ruff_cache/
35
+ .pytest_cache/
36
+ .ipynb_checkpoints/
37
+
38
+ # Claude Code context (per-machine, not for the public repo)
39
+ CLAUDE.md
40
+ CLAUDE.local.md
41
+ .claude/
42
+
43
+ # legacy / intermediate Prithvi artifacts (not shipped)
44
+ data/hls_stack_*.tif
45
+ data/prithvi_runs/
46
+ data/*.legacy_*
47
+ web/svelte/node_modules/
48
+ web/sveltekit/node_modules/
49
+ web/sveltekit/.svelte-kit/
50
+ # web/sveltekit/build/ (uncommented to allow deployment to HF Space)
51
+ # web/sveltekit/build/
52
+
53
+ # Experiments β€” cached HF model downloads, training artifacts, intermediate
54
+ # fixtures. RESULTS.md, NOTES.md, and source code stay tracked.
55
+ experiments/**/.cache/
56
+ experiments/**/restore/
57
+ experiments/**/publish/
58
+ experiments/**/*.tif
59
+ experiments/**/*.png
60
+ experiments/**/*.jpg
61
+ experiments/**/*.parquet
62
+ experiments/**/*.npy
63
+ pitch/screenshots-*/
64
+
65
+ # Marp deck render artefacts (regenerable via `make` in slides/)
66
+ slides/deck.pdf
67
+ slides/deck.html
68
+ slides/deck.pptx
69
+
70
+ # Session artifacts
71
+ /tmp/riprap-*
72
+ .deploy-state
73
+ *.bak
74
+ *.swp
75
+ *.swo
76
+ .playwright-mcp/
77
+
78
+ # Demo recordings (large; not committed)
79
+ assets/video/
80
+ slides/*.mp4
81
+ slides/asce/speaker_notes.md
82
+
83
+ # Local env overlays
84
+ .env.local
85
+ *.local.env
86
+
87
+ # Sensitive
88
+ AMD_TOKEN
89
+ submission.md
CHANGELOG.md ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Changelog
2
+
3
+ All notable changes to Riprap. The hackathon submission tag is
4
+ `v0.5.0` (build 2026-05-07); subsequent dates record polish work
5
+ that landed on the hackathon-period production deploys.
6
+
7
+ ## [Unreleased] β€” 2026-05-09 (Saturday)
8
+
9
+ ### Added
10
+ - **Per-query inference energy ledger** with real NVML readings off
11
+ the L4 GPU. The status row on the Findings region now reports
12
+ total Wh + total tokens for every briefing, with a leading icon
13
+ (`βœ“` / `◐` / `~`) disclosing whether the number was measured or
14
+ estimated. Full breakdown documented in
15
+ [`docs/EMISSIONS.md`](docs/EMISSIONS.md).
16
+ - `inference-vllm/proxy.py`: 100 ms-cadence NVML sampler, response
17
+ headers `X-GPU-Power-W` / `X-GPU-Energy-J` on every forwarded
18
+ POST, and a `GET /v1/power` endpoint for bracket-sampling clients.
19
+ - `app/emissions.py` β€” new module with a thread-local `Tracker` that
20
+ records every LLM and ML inference call (model, hardware, tokens,
21
+ duration, joules) with a `measured: bool` flag per row.
22
+ - `scripts/probe_stones_fire.py` β€” programmatic CI that runs an
23
+ address query against the lablab UI and asserts all five Stones
24
+ fire, no `torchvision::nms` / `deps unavailable` dep regression,
25
+ and the `emissions` block carries `nvidia_l4` hardware.
26
+ - `scripts/probe_benchmarks.py` β€” collects the canonical
27
+ four-address verification set into `outputs/benchmarks.json`
28
+ for the `docs/BENCHMARKS.md` page.
29
+ - `docs/EMISSIONS.md`, `docs/DEPLOY.md`, `docs/BENCHMARKS.md`,
30
+ `CHANGELOG.md`, `CONTRIBUTING.md`.
31
+
32
+ ### Changed
33
+ - The `RunHealthStrip` chip dropped the cloud-energy comparison
34
+ (the sign convention was misleading and the comparison is now
35
+ redundant given real measurements). New format:
36
+ `<icon> X.X Wh / Y.YK tok inference`.
37
+ - `app/llm.py:_default_hardware_label` defaults to `"NVIDIA L4"`
38
+ when remote vLLM is configured (was `"AMD MI300X"`, a stale
39
+ string from the droplet days).
40
+ - `app/llm.py:chat()` now brackets every completion with two GETs
41
+ to the inference Space's `/v1/power` endpoint; the average powers
42
+ the LLM-call energy reading instead of the data-sheet estimate.
43
+ - `app/inference.py:_post()` reads NVML headers off the proxy
44
+ response and forwards real joules into `emissions.record_ml`.
45
+
46
+ ### Fixed
47
+ - `app/flood_layers/prithvi_live.py`: when the configured remote
48
+ inference call fails (`RemoteUnreachable`), the specialist no
49
+ longer falls through to the local terratorch path. The local
50
+ path crashes with `RuntimeError: operator torchvision::nms does
51
+ not exist` on the cpu-basic UI Space; surfacing a clean
52
+ `remote prithvi-pluvial unreachable` skip is correct.
53
+ - `app/context/terramind_nyc.py:_try_remote()`: returns a
54
+ `{"ok": False, "skipped": "remote terramind/<adapter>: ..."}`
55
+ sentinel on remote failure, instead of `None` which was
56
+ silently masked as `deps unavailable on this deployment`.
57
+ - `web/main.py`: explicit `/favicon.svg`, `/favicon.png`,
58
+ `/favicon.ico`, `/robots.txt` routes β€” they were 404-ing under
59
+ the SvelteKit SPA fallback because only `/_app` was mounted off
60
+ the build directory.
61
+
62
+ ### Documentation
63
+ - Full README rewrite reflecting the post-droplet L4 topology, the
64
+ new emissions feature, and updated repo structure. Hackathon
65
+ framing preserved.
66
+ - New `docs/DEPLOY.md` with the production topology, env-var
67
+ reference, and per-Space deploy commands.
68
+ - New `docs/EMISSIONS.md` documenting what's measured vs. estimated,
69
+ the NVML pipeline, and how to verify.
70
+
71
+ ### Infrastructure note
72
+ - The DigitalOcean MI300X droplet was decommissioned 2026-05-06.
73
+ All production inference now serves from `msradam/riprap-vllm`
74
+ (NVIDIA L4). The MI300X runbook is preserved in
75
+ [`docs/DROPLET-RUNBOOK.md`](docs/DROPLET-RUNBOOK.md) for anyone
76
+ reproducing the AMD-judging setup; setting
77
+ `RIPRAP_HARDWARE_LABEL=AMD MI300X` swaps the emissions profile
78
+ back when redeploying to that hardware.
79
+
80
+ ---
81
+
82
+ ## [v0.5.0] β€” 2026-05-07
83
+
84
+ Hackathon submission tag.
85
+
86
+ ### Added
87
+ - Five-Stone Burr FSM with Granite-native document-role messages
88
+ - Mellea four-check rejection sampling for the Capstone
89
+ - SvelteKit UI with SSE streaming, briefing prose, evidence-card
90
+ grid, MapLibre overlay, citation drawer
91
+ - Three NYC-specialised foundation models published Apache-2.0:
92
+ `msradam/TerraMind-NYC-Adapters` (LULC + Buildings + TiM LoRAs),
93
+ `msradam/Prithvi-EO-2.0-NYC-Pluvial`,
94
+ `msradam/Granite-TTM-r2-Battery-Surge`
95
+ - 30+ FSM specialists across hazard memory, asset registers, live
96
+ observation, forecasting, and citation-grounded synthesis
CODE_OF_CONDUCT.md ADDED
@@ -0,0 +1,85 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ # Contributor Covenant Code of Conduct
3
+
4
+ ## Our Pledge
5
+
6
+ We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, caste, color, religion, or sexual identity and orientation.
7
+
8
+ We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community.
9
+
10
+ ## Our Standards
11
+
12
+ Examples of behavior that contributes to a positive environment for our community include:
13
+
14
+ * Demonstrating empathy and kindness toward other people
15
+ * Being respectful of differing opinions, viewpoints, and experiences
16
+ * Giving and gracefully accepting constructive feedback
17
+ * Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience
18
+ * Focusing on what is best not just for us as individuals, but for the overall community
19
+
20
+ Examples of unacceptable behavior include:
21
+
22
+ * The use of sexualized language or imagery, and sexual attention or advances of any kind
23
+ * Trolling, insulting or derogatory comments, and personal or political attacks
24
+ * Public or private harassment
25
+ * Publishing others' private information, such as a physical or email address, without their explicit permission
26
+ * Other conduct which could reasonably be considered inappropriate in a professional setting
27
+
28
+ ## Enforcement Responsibilities
29
+
30
+ Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful.
31
+
32
+ Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate.
33
+
34
+ ## Scope
35
+
36
+ This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event.
37
+
38
+ ## Enforcement
39
+
40
+ Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at msrahmanadam@gmail.com. All complaints will be reviewed and investigated promptly and fairly.
41
+
42
+ All community leaders are obligated to respect the privacy and security of the reporter of any incident.
43
+
44
+ ## Enforcement Guidelines
45
+
46
+ Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct:
47
+
48
+ ### 1. Correction
49
+
50
+ **Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community.
51
+
52
+ **Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested.
53
+
54
+ ### 2. Warning
55
+
56
+ **Community Impact**: A violation through a single incident or series of actions.
57
+
58
+ **Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban.
59
+
60
+ ### 3. Temporary Ban
61
+
62
+ **Community Impact**: A serious violation of community standards, including sustained inappropriate behavior.
63
+
64
+ **Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban.
65
+
66
+ ### 4. Permanent Ban
67
+
68
+ **Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals.
69
+
70
+ **Consequence**: A permanent ban from any sort of public interaction within the community.
71
+
72
+ ## Attribution
73
+
74
+ This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.1, available at [https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
75
+
76
+ Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder][Mozilla CoC].
77
+
78
+ For answers to common questions about this code of conduct, see the FAQ at [https://www.contributor-covenant.org/faq][FAQ]. Translations are available at [https://www.contributor-covenant.org/translations][translations].
79
+
80
+ [homepage]: https://www.contributor-covenant.org
81
+ [v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
82
+ [Mozilla CoC]: https://github.com/mozilla/diversity
83
+ [FAQ]: https://www.contributor-covenant.org/faq
84
+ [translations]: https://www.contributor-covenant.org/translations
85
+
CONTRIBUTING.md ADDED
@@ -0,0 +1,127 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Contributing
2
+
3
+ Riprap is the hackathon submission for the AMD Γ— lablab.ai
4
+ Developer Hackathon, but the source ships under Apache 2.0 and is
5
+ intended to be reusable as a template for citation-grounded civic
6
+ AI in any flood-vulnerable region. Pull requests welcome.
7
+
8
+ ## Quickstart
9
+
10
+ Python 3.12 + `uv`:
11
+
12
+ ```bash
13
+ git clone https://github.com/msradam/riprap-nyc
14
+ cd riprap-nyc
15
+ uv venv && uv pip install -r requirements.txt
16
+ ```
17
+
18
+ SvelteKit (the build is committed; only rebuild when sources
19
+ change under `web/sveltekit/src`):
20
+
21
+ ```bash
22
+ cd web/sveltekit && npm ci && npm run build && cd ../..
23
+ ```
24
+
25
+ Run the dev server locally pointing at the production inference
26
+ Space (real Granite + EO models, real NVML energy readings):
27
+
28
+ ```bash
29
+ RIPRAP_LLM_PRIMARY=vllm \
30
+ RIPRAP_LLM_BASE_URL=https://msradam-riprap-vllm.hf.space/v1 \
31
+ RIPRAP_LLM_API_KEY=<token> \
32
+ RIPRAP_ML_BACKEND=remote \
33
+ RIPRAP_ML_BASE_URL=https://msradam-riprap-vllm.hf.space \
34
+ RIPRAP_ML_API_KEY=<token> \
35
+ .venv/bin/uvicorn web.main:app --host 127.0.0.1 --port 7860
36
+ ```
37
+
38
+ Or run pure-local with Ollama (no GPU readings; data-sheet estimate):
39
+
40
+ ```bash
41
+ ollama pull granite4.1:3b granite4.1:8b
42
+ .venv/bin/uvicorn web.main:app --host 127.0.0.1 --port 7860
43
+ ```
44
+
45
+ ## Verifying changes
46
+
47
+ Two probe scripts exercise the live deployment end-to-end:
48
+
49
+ ```bash
50
+ # All five Stones must fire on the canonical address; emissions
51
+ # block must carry nvidia_l4 hardware; no torchvision/terratorch
52
+ # dep regressions in the trace.
53
+ PYTHONPATH=. uv run python scripts/probe_stones_fire.py --timeout 600
54
+
55
+ # Full canonical suite β€” five NYC addresses, intent-aware checks,
56
+ # Mellea grounding budget, no specialist crashes.
57
+ .venv/bin/python scripts/probe_addresses.py \
58
+ --base https://lablab-ai-amd-developer-hackathon-riprap-nyc.hf.space
59
+ ```
60
+
61
+ Both default to the lablab UI Space; pass `--base http://127.0.0.1:7860`
62
+ to hit a local server.
63
+
64
+ ## Structure
65
+
66
+ ```
67
+ app/ Python package β€” the FSM and its specialists
68
+ β”œβ”€β”€ fsm.py Burr FSM, one @action per probe
69
+ β”œβ”€β”€ llm.py LiteLLM Router shim (Ollama / vLLM)
70
+ β”œβ”€β”€ inference.py HTTP client for the riprap-models service
71
+ β”œβ”€β”€ emissions.py Per-query energy + token tracker
72
+ β”œβ”€β”€ stones/ Stone taxonomy (NAME / TAGLINE / collect())
73
+ β”œβ”€β”€ flood_layers/ Cornerstone probes (sandy, dep, microtopo, …)
74
+ β”œβ”€β”€ context/ Keystone + Touchstone register + EO probes
75
+ β”œβ”€β”€ live/ Lodestone forecast probes
76
+ β”œβ”€β”€ intents/ single_address / neighborhood / compare / live_now
77
+ β”œβ”€β”€ reconcile.py Capstone β€” Granite-native document reconcile
78
+ └── mellea_validator.py Mellea four-check rejection sampling
79
+
80
+ web/ FastAPI + SvelteKit
81
+ β”œβ”€β”€ main.py FastAPI app, SSE streaming, layer endpoints
82
+ β”œβ”€β”€ sveltekit/ Primary UI (adapter-static; build committed)
83
+ └── static/ Legacy custom-element pages (still mounted)
84
+
85
+ inference-vllm/ Inference Space source (vLLM + EO models + proxy)
86
+ β”œβ”€β”€ Dockerfile L4 image, bakes Granite 4.1 8B FP8 + EO deps
87
+ β”œβ”€β”€ entrypoint.sh Boots vllm, riprap-models, proxy as subprocesses
88
+ └── proxy.py Bearer-auth + NVML power sampler + SSE pass-through
89
+
90
+ inference/ Ollama-backed inference Space (fallback variant)
91
+ services/riprap-models/ The EO/forecast specialist HTTP service
92
+
93
+ scripts/
94
+ β”œβ”€β”€ probe_stones_fire.py Programmatic Stone-fire CI
95
+ β”œβ”€β”€ probe_addresses.py Canonical 5-address suite
96
+ β”œβ”€β”€ deploy_vllm_space.sh Deploy the L4 inference Space
97
+ β”œβ”€β”€ deploy_personal_space.sh Deploy the personal L4 mirror
98
+ β”œβ”€β”€ deploy_inference_space.sh Deploy the Ollama-backed inference Space
99
+ └── … Register builders, raster bakers, etc.
100
+
101
+ experiments/ Reproduction recipes for the three NYC fine-tunes
102
+ docs/ Architecture, methodology, deploy, emissions, runbooks
103
+ tests/ pytest suite (envelope + compare-shape tests)
104
+ ```
105
+
106
+ ## Style
107
+
108
+ - Python 3.12; `uv` for package management.
109
+ - LLM calls go through `app/llm.py` β€” never import `litellm` /
110
+ `ollama` directly from a specialist. The `chat()` shim wraps both
111
+ backends and the energy ledger reads off it.
112
+ - Remote ML calls go through `app/inference.py::_post`. Specialists
113
+ may try local fallback only when `inference.remote_enabled()` is
114
+ False; once a remote call has been attempted, return a clean
115
+ `{ok: False, skipped: ...}` on failure rather than crashing
116
+ through to local code paths that may not be installed.
117
+ - Every specialist emits one trace record per call with `step` /
118
+ `ok` / `elapsed_s` / `result` / `err` so the SSE stream and the
119
+ emissions tracker can reason about it.
120
+
121
+ ## Reporting issues
122
+
123
+ GitHub issues at <https://github.com/msradam/riprap-nyc/issues>.
124
+ For hackathon-period demo issues during May 4–10 2026, the live
125
+ deploy at
126
+ <https://lablab-ai-amd-developer-hackathon-riprap-nyc.hf.space>
127
+ is the source of truth.
Dockerfile ADDED
@@ -0,0 +1,104 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Riprap β€” Hugging Face Spaces deployment for the personal Space
2
+ # (msradam/riprap-nyc) on L4 hardware.
3
+ #
4
+ # Differences from the canonical Dockerfile:
5
+ #
6
+ # 1. L4 has 24 GB VRAM (vs 16 GB on T4 small), so we co-host the
7
+ # riprap-models service inside the same container instead of
8
+ # proxying to the AMD MI300X droplet. No external dependency.
9
+ #
10
+ # 2. We bake granite4.1:8b at *build* time. The build sandbox could
11
+ # not previously fit Granite + EO toolchain together; this Dockerfile
12
+ # keeps the EO install at runtime (entrypoint.l4.sh) and frees the
13
+ # sandbox budget for the 8B pull.
14
+ #
15
+ # 3. CUDA + ROCm-free torch β€” the inline riprap-models service uses
16
+ # the cu124 wheels installed via requirements.txt + the additional
17
+ # delta in services/riprap-models/requirements.txt.
18
+ #
19
+ # DO NOT push this image to the lablab Space β€” that one stays pointed
20
+ # at the MI300X droplet for AMD-judging continuity.
21
+
22
+ FROM nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04 AS base
23
+
24
+ ENV DEBIAN_FRONTEND=noninteractive
25
+ RUN apt-get update && apt-get install -y --no-install-recommends \
26
+ python3 python3-pip python3-venv python-is-python3 \
27
+ curl ca-certificates zstd procps git \
28
+ gdal-bin libgdal-dev libgeos-dev libproj-dev \
29
+ libgl1 libglib2.0-0 \
30
+ && rm -rf /var/lib/apt/lists/*
31
+
32
+ RUN useradd -m -u 1000 user
33
+ ENV HOME=/home/user \
34
+ PATH=/home/user/.local/bin:/usr/local/bin:/usr/bin:/bin \
35
+ PYTHONUNBUFFERED=1 \
36
+ HF_HOME=/home/user/.cache/huggingface \
37
+ OLLAMA_HOST=127.0.0.1:11434 \
38
+ OLLAMA_NUM_PARALLEL=1 \
39
+ OLLAMA_KEEP_ALIVE=24h \
40
+ OLLAMA_MAX_LOADED_MODELS=2 \
41
+ OLLAMA_FLASH_ATTENTION=1 \
42
+ OLLAMA_KV_CACHE_TYPE=q8_0 \
43
+ OLLAMA_DEBUG=1 \
44
+ OLLAMA_MODELS=/home/user/.ollama/models \
45
+ RIPRAP_OLLAMA_3B_TAG=granite4.1:8b \
46
+ RIPRAP_LLM_PRIMARY=ollama \
47
+ RIPRAP_LLM_BASE_URL=http://127.0.0.1:11434/v1 \
48
+ RIPRAP_ML_BACKEND=remote \
49
+ RIPRAP_ML_BASE_URL=http://127.0.0.1:7861
50
+
51
+ RUN curl -fsSL https://ollama.com/install.sh | sh
52
+
53
+ WORKDIR /home/user/app
54
+
55
+ # Web app deps (torch cu124 lands via sentence-transformers / etc.).
56
+ COPY --chown=user:user requirements.txt ./
57
+ RUN pip install --no-cache-dir --upgrade pip && \
58
+ pip install --no-cache-dir -r requirements.txt
59
+
60
+ # riprap-models delta deps. Use the existing requirements.txt at the
61
+ # *service* level, but skip requirements-full.txt β€” its ROCm-frozen
62
+ # torch pin would clobber the cu124 wheels installed above.
63
+ COPY --chown=user:user services/riprap-models/requirements.txt /tmp/req-models.txt
64
+ RUN pip install --no-cache-dir -r /tmp/req-models.txt
65
+
66
+ # Bake torchvision (CUDA 12.4 wheel) and peft at build time. The
67
+ # canonical entrypoint.sh runtime-installs torchvision via the EO
68
+ # toolchain path because the canonical CPU Space's build sandbox is
69
+ # too tight; L4 builds have more room, and a properly matched
70
+ # torchvision avoids the `torchvision::nms does not exist` runtime
71
+ # error the canonical setup hits. peft is required by the riprap-
72
+ # models service for the TerraMind LoRA inference path.
73
+ RUN pip install --no-cache-dir \
74
+ --index-url https://download.pytorch.org/whl/cu124 \
75
+ torchvision \
76
+ && pip install --no-cache-dir peft==0.18.1
77
+
78
+ # Bake Granite 4.1 weights into the image (EO toolchain is installed
79
+ # at runtime β€” see entrypoint.l4.sh β€” to keep the build sandbox under
80
+ # its disk threshold).
81
+ RUN mkdir -p $OLLAMA_MODELS && \
82
+ ollama serve & \
83
+ OPID=$! && \
84
+ for i in $(seq 1 30); do curl -sf http://127.0.0.1:11434/ > /dev/null && break; sleep 1; done && \
85
+ ollama pull granite4.1:8b && \
86
+ kill $OPID 2>/dev/null || true && \
87
+ sleep 2
88
+
89
+ # App code, fixtures, and inline model service.
90
+ COPY --chown=user:user app/ ./app/
91
+ COPY --chown=user:user web/ ./web/
92
+ COPY --chown=user:user scripts/ ./scripts/
93
+ COPY --chown=user:user data/ ./data/
94
+ COPY --chown=user:user corpus/ ./corpus/
95
+ COPY --chown=user:user services/riprap-models/main.py ./riprap_models.py
96
+ COPY --chown=user:user agent.py riprap.py ./
97
+ COPY --chown=user:user entrypoint.sh ./entrypoint.sh
98
+ RUN chmod +x ./entrypoint.sh
99
+
100
+ RUN chown -R user:user /home/user
101
+ USER user
102
+
103
+ EXPOSE 7860
104
+ CMD ["./entrypoint.sh"]
README.md ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: Riprap NYC (Personal Mirror, L4)
3
+ emoji: 🌊
4
+ colorFrom: blue
5
+ colorTo: indigo
6
+ sdk: docker
7
+ pinned: false
8
+ short_description: NYC flood-exposure briefings on L4 (self-contained).
9
+ ---
10
+
11
+ # Riprap β€” NYC flood-exposure briefings (L4 self-contained mirror)
12
+
13
+ This Space is a self-contained mirror of
14
+ [`github.com/msradam/riprap-nyc`](https://github.com/msradam/riprap-nyc).
15
+
16
+ It runs on a single L4 GPU and co-hosts everything in one container:
17
+ Granite 4.1 8B (via Ollama), Prithvi-EO 2.0 NYC-Pluvial, TerraMind
18
+ LULC + Buildings LoRAs, and Granite TTM r2 β€” no external droplet
19
+ dependency. Sleeps on idle; first request after sleep takes ~45–60 s
20
+ to wake.
21
+
22
+ The hackathon submission Space (CPU UI, droplet proxy) lives at
23
+ [`AMD-hackathon/riprap-nyc`](https://lablab-ai-amd-developer-hackathon-riprap-nyc.hf.space).
24
+
25
+ Apache 2.0. See the GitHub repo for full source, architecture
26
+ deep-dive, methodology, and licence map.
SECURITY.md ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Security policy
2
+
3
+ ## Reporting a vulnerability
4
+
5
+ If you find a security issue in Riprap, please report it privately so
6
+ it can be triaged before disclosure.
7
+
8
+ - Email: **msrahmanadam@gmail.com** (subject prefix: `[riprap-security]`)
9
+ - Or open a [GitHub Security Advisory](https://github.com/msradam/riprap-nyc/security/advisories/new)
10
+ on this repository.
11
+
12
+ Please do not file a public GitHub issue for security reports.
13
+
14
+ We aim to acknowledge reports within 72 hours and to ship a fix or a
15
+ mitigation plan within two weeks of triage. If the report concerns a
16
+ vulnerability in an upstream model or service Riprap depends on
17
+ (IBM Granite, vLLM, Hugging Face Spaces, NYC Open Data endpoints), we
18
+ will help coordinate disclosure with the upstream maintainer.
19
+
20
+ ## Threat-surface notes
21
+
22
+ Riprap is a citation-grounded synthesis layer over public-record
23
+ data. By design, the runtime:
24
+
25
+ - contacts only **public-record APIs** (NYC Open Data, FloodNet,
26
+ USGS, NOAA, NWS, NYS DOH, MTA, NYCHA, NYC DOE, OpenStreetMap /
27
+ Nominatim) and the configured inference Spaces;
28
+ - does **not** authenticate against user accounts or store
29
+ user-identifying data β€” the address bar is the only input;
30
+ - runs the SvelteKit UI as a static SPA over a FastAPI backend
31
+ with no persistent database.
32
+
33
+ The vulnerability surface is therefore small. Plausible categories
34
+ worth a report:
35
+
36
+ - Prompt-injection paths via document content that escape the
37
+ Mellea grounding loop and surface unverifiable claims as cited.
38
+ - SSRF / abuse via crafted address strings that drive backend
39
+ HTTP calls to unintended hosts.
40
+ - Token leakage in proxy headers or SSE streams
41
+ (`inference-vllm/proxy.py`, `web/main.py`).
42
+ - Denial-of-service patterns that exceed the hosted Space's
43
+ resource budget.
44
+ - Supply-chain issues in pinned deps (`requirements.txt`,
45
+ `web/sveltekit/package.json`).
46
+
47
+ ## Out of scope
48
+
49
+ - Self-hosted deployments running with custom configuration or
50
+ custom datasets β€” please file those as regular bugs.
51
+ - Findings that require physical or local-network access to a
52
+ user's machine.
53
+ - Issues in the lablab.ai or Hugging Face Spaces hosting platforms
54
+ themselves; please report those upstream.
agent.py ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Riprap agent CLI β€” address β†’ cited briefing via the Burr FSM.
2
+
3
+ Usage:
4
+ python agent.py "180 Beach 35 St, Queens"
5
+ python agent.py "280 Broome St, Manhattan" --json
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import argparse
10
+ import json
11
+ import sys
12
+ import warnings
13
+
14
+ warnings.filterwarnings("ignore")
15
+
16
+ from app.fsm import run # noqa: E402
17
+
18
+
19
+ def main() -> int:
20
+ ap = argparse.ArgumentParser()
21
+ ap.add_argument("query", help="NYC address or natural-language location")
22
+ ap.add_argument("--json", action="store_true", help="emit full JSON state")
23
+ args = ap.parse_args()
24
+
25
+ print(f"\n query: {args.query}", file=sys.stderr)
26
+ print(" running FSM... (Granite 4.1 + open data, all local)\n", file=sys.stderr)
27
+
28
+ result = run(args.query)
29
+
30
+ if args.json:
31
+ print(json.dumps(result, indent=2, default=str))
32
+ return 0
33
+
34
+ print("─── trace " + "─" * 56)
35
+ for step in result["trace"]:
36
+ ok = "βœ“" if step["ok"] else "βœ—"
37
+ line = f" {ok} {step['step']:22s} {step.get('elapsed_s', 0):>5.2f}s"
38
+ if step.get("result"):
39
+ line += " " + json.dumps(step["result"], default=str)
40
+ elif step.get("err"):
41
+ line += " ERR: " + step["err"]
42
+ print(line)
43
+
44
+ print("\n─── cited report " + "─" * 49)
45
+ print()
46
+ print(result["paragraph"])
47
+ print()
48
+ return 0
49
+
50
+
51
+ if __name__ == "__main__":
52
+ sys.exit(main())
app/__init__.py ADDED
File without changes
app/areas/__init__.py ADDED
File without changes
app/areas/nta.py ADDED
@@ -0,0 +1,224 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """NYC Neighborhood Tabulation Area (NTA 2020) resolver.
2
+
3
+ NTAs are NYC Department of City Planning's official neighborhood unit:
4
+ ~262 polygons covering all 5 boroughs, including some park / airport
5
+ slivers. They are the canonical "neighborhood" unit for NYC civic data.
6
+
7
+ This module provides:
8
+ - load() β†’ GeoDataFrame with all NTAs (cached)
9
+ - resolve(name) β†’ list of matching NTAs by fuzzy name match, or by borough
10
+ - by_code(code) β†’ exact lookup
11
+ - polygon_for(code) β†’ shapely Polygon in EPSG:4326
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import re
16
+ from functools import lru_cache
17
+ from pathlib import Path
18
+ from typing import Any
19
+
20
+ import geopandas as gpd
21
+ from shapely.geometry import Polygon
22
+
23
+ DATA_PATH = Path(__file__).resolve().parents[2] / "data" / "nyc_ntas_2020.geojson"
24
+
25
+ # Common alias map: user-typed strings β†’ canonical NTA names. We don't need to
26
+ # be exhaustive here; the fuzzy matcher catches most cases. This handles the
27
+ # few hard ones where the official NTA name differs from local usage.
28
+ ALIASES = {
29
+ "the rockaways": "Rockaway Beach-Arverne-Edgemere",
30
+ "rockaway": "Rockaway Beach-Arverne-Edgemere",
31
+ "brighton": "Brighton Beach",
32
+ "lower east side": "Lower East Side",
33
+ "les": "Lower East Side",
34
+ "soho": "SoHo-Little Italy-Hudson Square",
35
+ "tribeca": "Tribeca-Civic Center",
36
+ "fidi": "Financial District-Battery Park City",
37
+ "downtown brooklyn":"Downtown Brooklyn-DUMBO-Boerum Hill",
38
+ "dumbo": "Downtown Brooklyn-DUMBO-Boerum Hill",
39
+ "park slope": "Park Slope",
40
+ "carroll gardens": "Carroll Gardens-Cobble Hill-Gowanus-Red Hook",
41
+ "red hook": "Carroll Gardens-Cobble Hill-Gowanus-Red Hook",
42
+ "gowanus": "Carroll Gardens-Cobble Hill-Gowanus-Red Hook",
43
+ "hollis": "Queens Village-Hollis-Bellerose",
44
+ "long island city": "Hunters Point-Sunnyside-West Maspeth",
45
+ "lic": "Hunters Point-Sunnyside-West Maspeth",
46
+ "astoria": "Astoria (Central)",
47
+ "flushing": "Flushing-Willets Point",
48
+ "harlem": "Central Harlem (North)",
49
+ "east harlem": "East Harlem (North)",
50
+ "washington heights":"Washington Heights (North)",
51
+ "midtown": "Midtown South-Flatiron-Union Square",
52
+ "upper east side": "Upper East Side-Carnegie Hill",
53
+ "ues": "Upper East Side-Carnegie Hill",
54
+ "upper west side": "Upper West Side-Lincoln Square",
55
+ "uws": "Upper West Side-Lincoln Square",
56
+ "coney island": "Coney Island-Sea Gate",
57
+ }
58
+
59
+ BOROUGH_NORMALIZE = {
60
+ "manhattan": "Manhattan", "mn": "Manhattan",
61
+ "brooklyn": "Brooklyn", "bk": "Brooklyn", "kings": "Brooklyn",
62
+ "queens": "Queens", "qn": "Queens",
63
+ "bronx": "Bronx", "the bronx": "Bronx", "bx": "Bronx",
64
+ "staten island": "Staten Island", "si": "Staten Island", "richmond": "Staten Island",
65
+ }
66
+
67
+
68
+ def _normalize(s: str) -> str:
69
+ return re.sub(r"[^a-z]+", "", (s or "").lower())
70
+
71
+
72
+ @lru_cache(maxsize=1)
73
+ def load() -> gpd.GeoDataFrame:
74
+ """Load the NTA 2020 GeoJSON; coerce CRS to EPSG:4326. Cached."""
75
+ g = gpd.read_file(DATA_PATH)
76
+ if g.crs is None or g.crs.to_string() != "EPSG:4326":
77
+ g = g.to_crs("EPSG:4326")
78
+ return g
79
+
80
+
81
+ def by_code(code: str) -> dict | None:
82
+ g = load()
83
+ hit = g[g["nta2020"] == code]
84
+ if hit.empty:
85
+ return None
86
+ return _row_to_dict(hit.iloc[0])
87
+
88
+
89
+ def _row_to_dict(row) -> dict:
90
+ return {
91
+ "nta_code": row["nta2020"],
92
+ "nta_name": row["ntaname"],
93
+ "borough": row["boroname"],
94
+ "cdta": row.get("cdtaname"),
95
+ "geometry": row["geometry"],
96
+ }
97
+
98
+
99
+ def borough_match(query: str) -> str | None:
100
+ """If query matches a borough name (or common abbreviation), return the
101
+ canonical name. Otherwise return None."""
102
+ q = query.strip().lower()
103
+ return BOROUGH_NORMALIZE.get(q)
104
+
105
+
106
+ def resolve(query: str) -> list[dict[str, Any]]:
107
+ """Resolve a free-text query to NTA(s).
108
+
109
+ Strategy (in priority order):
110
+ 1. Borough match β†’ all NTAs in borough.
111
+ 2. Alias map β†’ exact NTA name match.
112
+ 3. Case-insensitive EXACT name match (so 'Kew Gardens' wins over
113
+ 'Kew Gardens Hills' when both exist).
114
+ 4. Substring match on normalized NTA name. When multiple match,
115
+ prefer the one whose normalized name length is closest to the
116
+ query β€” avoids 'Kew Gardens' resolving to 'Kew Gardens Hills'.
117
+ 5. CDTA-name substring fallback.
118
+ """
119
+ g = load()
120
+ q = (query or "").strip()
121
+ if not q:
122
+ return []
123
+ boro = borough_match(q)
124
+ if boro:
125
+ hits = g[g["boroname"] == boro]
126
+ return [_row_to_dict(r) for _, r in hits.iterrows()]
127
+
128
+ alias = ALIASES.get(q.lower())
129
+ if alias:
130
+ hits = g[g["ntaname"] == alias]
131
+ if not hits.empty:
132
+ return [_row_to_dict(r) for _, r in hits.iterrows()]
133
+
134
+ # Exact (case-insensitive) β€” preferred over substring
135
+ name_lower = g["ntaname"].fillna("").str.lower()
136
+ exact = g[name_lower == q.lower()]
137
+ if not exact.empty:
138
+ return [_row_to_dict(r) for _, r in exact.iterrows()]
139
+
140
+ qn = _normalize(q)
141
+ if not qn:
142
+ return []
143
+ name_norm = g["ntaname"].fillna("").map(_normalize)
144
+ contains = g[name_norm.str.contains(qn, na=False)].copy()
145
+ if not contains.empty:
146
+ contains["_diff"] = contains["ntaname"].fillna("").map(
147
+ lambda s: abs(len(_normalize(s)) - len(qn))
148
+ )
149
+ contains = contains.sort_values("_diff")
150
+ return [_row_to_dict(r) for _, r in contains.iterrows()]
151
+
152
+ cdta_norm = g["cdtaname"].fillna("").map(_normalize)
153
+ contains = g[cdta_norm.str.contains(qn, na=False)]
154
+ if not contains.empty:
155
+ return [_row_to_dict(r) for _, r in contains.iterrows()]
156
+
157
+ return []
158
+
159
+
160
+ def polygon_for(code: str) -> Polygon | None:
161
+ hit = by_code(code)
162
+ return hit["geometry"] if hit else None
163
+
164
+
165
+ def resolve_from_text(text: str) -> list[dict[str, Any]]: # TODO(cleanup): cc-grade-D (25)
166
+ """Scan free-text (e.g. a full natural-language query) for any known NTA
167
+ name, alias, or borough. Returns the first match. This is the fallback
168
+ when the planner failed to extract a clean target.
169
+
170
+ Strategy: walk ALIASES first (cheap), then iterate NTA names and look
171
+ for the longest match contained in the text. We prefer the longest
172
+ match so 'Carroll Gardens' wins over 'Gardens'.
173
+ """
174
+ t = (text or "").lower()
175
+ if not t:
176
+ return []
177
+ # Boroughs first (whole-word-ish β€” avoid false hits inside "queensland" etc.)
178
+ for boro_key, canon in BOROUGH_NORMALIZE.items():
179
+ if f" {boro_key} " in f" {t} " or t.startswith(boro_key + " ") or t.endswith(" " + boro_key):
180
+ hits = resolve(canon)
181
+ if hits:
182
+ return hits
183
+ # Alias keys, longest first
184
+ for key in sorted(ALIASES.keys(), key=len, reverse=True):
185
+ if key in t:
186
+ hits = resolve(key)
187
+ if hits:
188
+ return hits
189
+ # NTA names. Order: longest first so multi-word names match before
190
+ # shorter substrings, AND preferring the WORD-BOUNDARY match so
191
+ # "Kew Gardens" in the query doesn't collide with "Kew Gardens Hills"
192
+ # (the latter is longer; without word-boundary checking it'd match
193
+ # nothing, but with substring-in-text it'd match if the query ever
194
+ # contained the longer phrase). Caller picks the closest-length match.
195
+ g = load()
196
+ names = sorted(set(g["ntaname"].dropna().str.lower().tolist()), key=len, reverse=True)
197
+ matches = []
198
+ for name in names:
199
+ if not name or len(name) < 4:
200
+ continue
201
+ # Word-boundary-ish check: name must appear bounded by start/end or
202
+ # whitespace/punct (so "kew gardens hills" matches but "kew gardens"
203
+ # alone doesn't trigger "kew gardens hills" because of the trailing
204
+ # space requirement).
205
+ padded_t = f" {t} "
206
+ if f" {name} " in padded_t or f" {name}." in padded_t or f" {name}," in padded_t or f" {name}?" in padded_t:
207
+ matches.append(name)
208
+ if matches:
209
+ # Prefer the longest word-boundary match β€” most specific.
210
+ best = sorted(matches, key=len, reverse=True)[0]
211
+ hits = resolve(best)
212
+ if hits:
213
+ return hits
214
+ # Fallback: any substring (no boundary). Less precise, but catches
215
+ # casual queries like "show me red hook" where "red hook" is a
216
+ # neighborhood-name fragment within a longer NTA name.
217
+ for name in names:
218
+ if not name or len(name) < 4:
219
+ continue
220
+ if name in t:
221
+ hits = resolve(name)
222
+ if hits:
223
+ return hits
224
+ return []
app/assets/__init__.py ADDED
File without changes
app/assets/mta_entrances.py ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """MTA Subway Entrances and Exits (NY OpenData i9wp-a4ja).
2
+
3
+ ~1,900 subway entrances city-wide. The MTA Climate Resilience Roadmap
4
+ (Oct 2025) names ~1,500 of these as priorities for sealing β€” this is
5
+ exactly the asset class our RAG corpus has the most to say about, and
6
+ exactly the audience (MTA capital planners, transit advocacy) the
7
+ register is built for.
8
+ """
9
+ from __future__ import annotations
10
+
11
+ from pathlib import Path
12
+
13
+ import geopandas as gpd
14
+ import httpx
15
+
16
+ from app.spatial import DATA, NYC_CRS
17
+
18
+ URL = "https://data.ny.gov/api/geospatial/i9wp-a4ja?method=export&format=GeoJSON"
19
+ LOCAL = DATA / "mta_entrances.geojson"
20
+
21
+
22
+ def _ensure_fixture() -> Path:
23
+ if LOCAL.exists():
24
+ return LOCAL
25
+ print("downloading MTA Subway Entrances (one-time)...", flush=True)
26
+ r = httpx.get(URL, timeout=60)
27
+ r.raise_for_status()
28
+ LOCAL.write_text(r.text)
29
+ return LOCAL
30
+
31
+
32
+ def load() -> gpd.GeoDataFrame:
33
+ _ensure_fixture()
34
+ g = gpd.read_file(LOCAL)
35
+ if g.crs is None:
36
+ g.set_crs("EPSG:4326", inplace=True)
37
+ g = g.to_crs(NYC_CRS)
38
+ rename_map = {
39
+ "stop_name": "name",
40
+ "constrained_floor_to_floor_height": None,
41
+ "borough": "borough",
42
+ "entrance_type": "entrance_type",
43
+ "ada": "ada",
44
+ "north_south_street": "ns_street",
45
+ "east_west_street": "ew_street",
46
+ "corner": "corner",
47
+ }
48
+ for k, v in rename_map.items():
49
+ if v and k in g.columns and k != v:
50
+ g = g.rename(columns={k: v})
51
+
52
+ # build a usable address-style label
53
+ def label(row):
54
+ nm = (row.get("name") or "").strip()
55
+ ns = (row.get("ns_street") or "").strip()
56
+ ew = (row.get("ew_street") or "").strip()
57
+ cn = (row.get("corner") or "").strip()
58
+ bits = [nm]
59
+ cross = " & ".join(b for b in [ns, ew] if b)
60
+ if cross: bits.append(cross)
61
+ if cn: bits.append(f"({cn})")
62
+ return ", ".join([b for b in bits if b])
63
+
64
+ g["address"] = g.apply(label, axis=1)
65
+ if "borough" in g.columns:
66
+ boro_map = {"M": "Manhattan", "Bk": "Brooklyn", "B": "Brooklyn",
67
+ "Q": "Queens", "Bx": "Bronx", "SI": "Staten Island"}
68
+ g["borough"] = g["borough"].astype(str).map(lambda v: boro_map.get(v, v.title()))
69
+
70
+ keep = [c for c in ["name", "address", "borough", "entrance_type",
71
+ "ada", "ns_street", "ew_street", "corner", "geometry"]
72
+ if c in g.columns]
73
+ return g[keep].copy()
app/assets/nycha.py ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """NYCHA Developments (NYC OpenData phvi-damg).
2
+
3
+ 326 public-housing developments across NYC. Used as an asset class for
4
+ the bulk-mode register; the parent rationale for surfacing this layer
5
+ is that NYCHA was hit hard by Sandy and remains a published Tier-1
6
+ flood-resilience priority in the city's Hazard Mitigation Plan.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import geopandas as gpd
11
+
12
+ from app.spatial import DATA, load_layer
13
+
14
+
15
+ def load() -> gpd.GeoDataFrame:
16
+ g = load_layer(DATA / "nycha.geojson")
17
+ # NYCHA developments come back as polygons; the FSM expects point
18
+ # geometry for spatial joins. Use centroid.
19
+ g = g.copy()
20
+ g["geometry"] = g.geometry.centroid
21
+
22
+ # NYCHA Developments has only `developmen` (truncated label), tds_num, borough.
23
+ g = g.rename(columns={"developmen": "name"})
24
+ g["address"] = g["name"] # the field doubles as both
25
+ g["borough"] = g["borough"].str.title() # "BRONX" -> "Bronx" to match Riprap convention
26
+
27
+ keep = [c for c in ["name", "address", "borough", "tds_num", "geometry"] if c in g.columns]
28
+ return g[keep].copy()
app/assets/schools.py ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """NYC DOE School Point Locations (Socrata a3nt-yts4)."""
2
+ from __future__ import annotations
3
+
4
+ import geopandas as gpd
5
+
6
+ from app.spatial import DATA, load_layer
7
+
8
+ BORO = {"1": "Manhattan", "2": "Bronx", "3": "Brooklyn", "4": "Queens", "5": "Staten Island"}
9
+
10
+
11
+ def load() -> gpd.GeoDataFrame:
12
+ g = load_layer(DATA / "schools.geojson")
13
+ g = g.rename(columns={
14
+ "loc_code": "loc_code",
15
+ "loc_name": "name",
16
+ "address": "address",
17
+ "bbl": "bbl",
18
+ "bin": "bin",
19
+ "boronum": "boro_num",
20
+ "geodistric": "geo_district",
21
+ "adimindist": "admin_district",
22
+ })
23
+ g["borough"] = g["boro_num"].astype(str).map(BORO)
24
+ g["bbl"] = g["bbl"].astype(str).str.replace(r"\.0$", "", regex=True)
25
+ keep = ["loc_code", "name", "address", "borough", "bbl", "bin",
26
+ "geo_district", "admin_district", "geometry"]
27
+ return g[keep].copy()
app/context/__init__.py ADDED
File without changes
app/context/_polygonize.py ADDED
@@ -0,0 +1,165 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Vectorize a uint8 prediction raster (binary mask or class index)
2
+ into an EPSG:4326 GeoJSON FeatureCollection so the frontend can paint
3
+ it on the MapLibre map.
4
+
5
+ The droplet's `/v1/prithvi-pluvial` and `/v1/terramind` routes return
6
+ their predictions as base64-encoded uint8 with a shape and (where
7
+ relevant) a class-label list. This module reconstructs the affine
8
+ transform from the chip's geographic bounds (which the HF Space
9
+ already knows) and walks `rasterio.features.shapes` to build polygons
10
+ in the chip's native CRS, then reprojects to WGS84 for the map.
11
+
12
+ Best-effort: any failure returns an empty FeatureCollection rather
13
+ than raising into the caller's path. The map layer is decorative β€”
14
+ the briefing is the deliverable.
15
+ """
16
+ from __future__ import annotations
17
+
18
+ import base64
19
+ import logging
20
+
21
+ log = logging.getLogger("riprap.polygonize")
22
+
23
+ EMPTY: dict = {"type": "FeatureCollection", "features": []}
24
+
25
+
26
+ def _decode_pred(pred_b64: str, pred_shape: list[int]):
27
+ """Inverse of the droplet's `base64(pred.tobytes())`. Returns a
28
+ uint8 numpy array of shape `pred_shape`, or None on decode error."""
29
+ try:
30
+ import numpy as np
31
+ raw = base64.b64decode(pred_b64)
32
+ return np.frombuffer(raw, dtype="uint8").reshape(pred_shape)
33
+ except Exception:
34
+ log.exception("polygonize: pred decode failed")
35
+ return None
36
+
37
+
38
+ def polygonize_class_raster(
39
+ pred_b64: str,
40
+ pred_shape: list[int],
41
+ class_labels: list[str] | None,
42
+ bounds_4326: tuple[float, float, float, float],
43
+ *,
44
+ drop_classes: tuple[int, ...] = (0,),
45
+ simplify_tolerance: float = 0.0,
46
+ ) -> dict:
47
+ """Vectorize a categorical prediction raster (one integer class per
48
+ pixel) into a FeatureCollection with one Feature per connected
49
+ polygon. `bounds_4326` is `(minlon, minlat, maxlon, maxlat)` of the
50
+ chip; the raster is assumed to span those bounds at uniform
51
+ pixel size. Each feature carries `class_idx` and `class_label`
52
+ so the frontend can color by class.
53
+
54
+ `drop_classes`: skip pixels matching these class indices (default
55
+ drops 0 = "Background" / "outside" / etc).
56
+ """
57
+ pred = _decode_pred(pred_b64, pred_shape)
58
+ if pred is None:
59
+ return EMPTY
60
+ try:
61
+ from rasterio.features import shapes
62
+ from rasterio.transform import from_bounds
63
+ from shapely.geometry import shape
64
+ h, w = pred.shape
65
+ minlon, minlat, maxlon, maxlat = bounds_4326
66
+ # The chip is in EPSG:4326 for our use β€” Sentinel-2 chips are
67
+ # natively in their UTM zone, but we can polygonize against the
68
+ # WGS84 extent because the inference chip is a small bbox where
69
+ # the pixel-grid β†’ lat/lon mapping is locally affine (sub-pixel
70
+ # error at NYC scale).
71
+ transform = from_bounds(minlon, minlat, maxlon, maxlat, w, h)
72
+ feats = []
73
+ for geom, value in shapes(pred, mask=pred > 0, transform=transform):
74
+ v = int(value)
75
+ if v in drop_classes:
76
+ continue
77
+ label = (class_labels[v]
78
+ if class_labels and 0 <= v < len(class_labels)
79
+ else f"class_{v}")
80
+ poly = shape(geom)
81
+ if simplify_tolerance > 0:
82
+ poly = poly.simplify(simplify_tolerance, preserve_topology=True)
83
+ if poly.is_empty:
84
+ continue
85
+ feats.append({
86
+ "type": "Feature",
87
+ "geometry": poly.__geo_interface__,
88
+ "properties": {
89
+ "class_idx": v,
90
+ "class_label": label,
91
+ "fill_color": _PALETTE.get(label.lower(), _DEFAULT_FILL),
92
+ },
93
+ })
94
+ return {"type": "FeatureCollection", "features": feats}
95
+ except Exception:
96
+ log.exception("polygonize: class raster vectorisation failed")
97
+ return EMPTY
98
+
99
+
100
+ def polygonize_binary_mask(
101
+ pred_b64: str,
102
+ pred_shape: list[int],
103
+ bounds_4326: tuple[float, float, float, float],
104
+ *,
105
+ label: str = "water",
106
+ fill_color: str = "#4A90E2",
107
+ simplify_tolerance: float = 0.0,
108
+ ) -> dict:
109
+ """Vectorize a binary prediction raster (e.g. Prithvi water mask;
110
+ 1 = water, 0 = not). Returns one Feature per connected positive
111
+ region. Use this for prithvi_eo_live and the buildings LoRA."""
112
+ pred = _decode_pred(pred_b64, pred_shape)
113
+ if pred is None:
114
+ return EMPTY
115
+ try:
116
+ from rasterio.features import shapes
117
+ from rasterio.transform import from_bounds
118
+ from shapely.geometry import shape
119
+ h, w = pred.shape
120
+ minlon, minlat, maxlon, maxlat = bounds_4326
121
+ transform = from_bounds(minlon, minlat, maxlon, maxlat, w, h)
122
+ feats = []
123
+ for geom, _value in shapes(pred, mask=pred > 0, transform=transform):
124
+ poly = shape(geom)
125
+ if simplify_tolerance > 0:
126
+ poly = poly.simplify(simplify_tolerance, preserve_topology=True)
127
+ if poly.is_empty:
128
+ continue
129
+ feats.append({
130
+ "type": "Feature",
131
+ "geometry": poly.__geo_interface__,
132
+ "properties": {
133
+ "class_label": label,
134
+ "fill_color": fill_color,
135
+ },
136
+ })
137
+ return {"type": "FeatureCollection", "features": feats}
138
+ except Exception:
139
+ log.exception("polygonize: binary mask vectorisation failed")
140
+ return EMPTY
141
+
142
+
143
+ # Lightweight palette used by the LULC + buildings layers. Frontend
144
+ # may override via `fill_color` per feature; this is a sensible
145
+ # default keyed on lowercase class labels.
146
+ _DEFAULT_FILL = "#A0A0A0"
147
+ _PALETTE = {
148
+ # ESRI 2020 LULC schema (terramind v1 base generative)
149
+ "water": "#1F77B4",
150
+ "trees": "#2CA02C",
151
+ "grass": "#7FBF53",
152
+ "flooded vegetation": "#74C476",
153
+ "crops": "#E1C75A",
154
+ "scrub/shrub": "#A6BC44",
155
+ "built": "#D62728",
156
+ "bare ground": "#B07A4C",
157
+ "snow/ice": "#E0E7EC",
158
+ "clouds": "#CCCCCC",
159
+ # NYC LoRA LULC schema
160
+ "cropland": "#E1C75A",
161
+ "bare": "#B07A4C",
162
+ # Buildings LoRA
163
+ "building": "#D62728",
164
+ "background": _DEFAULT_FILL,
165
+ }
app/context/dob_permits.py ADDED
@@ -0,0 +1,258 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """NYC DOB construction-permit specialist β€” "what are they building".
2
+
3
+ Pulls active NYC DOB Permit Issuance records (Socrata `ipu4-2q9a`)
4
+ inside a polygon, filtered to recent New Building (NB), major
5
+ Alteration (A1), and Demolition (DM) jobs. Each project is then
6
+ cross-referenced against the static flood layers (Sandy 2012, DEP
7
+ Stormwater scenarios) so the reconciler can write things like:
8
+
9
+ "12 active major construction projects in Gowanus. Of these,
10
+ 8 sit inside the DEP Extreme-2080 stormwater scenario."
11
+
12
+ The dataset uses separate gis_latitude / gis_longitude columns rather
13
+ than a Socrata Point, so we bbox-filter via SoQL then do exact
14
+ point-in-polygon containment client-side with shapely.
15
+ """
16
+ from __future__ import annotations
17
+
18
+ import logging
19
+ from collections import Counter
20
+ from dataclasses import asdict, dataclass
21
+ from datetime import date, datetime, timedelta
22
+ from typing import Any
23
+
24
+ import geopandas as gpd
25
+ import httpx
26
+ from shapely.geometry import Point
27
+
28
+ log = logging.getLogger("riprap.dob_permits")
29
+
30
+ URL = "https://data.cityofnewyork.us/resource/ipu4-2q9a.json"
31
+ DOC_ID = "dob_permits"
32
+ CITATION = ("NYC DOB Permit Issuance (NYC OpenData ipu4-2q9a) β€” "
33
+ "issued/in-progress construction permits")
34
+
35
+ JOB_TYPE_LABELS = {
36
+ "NB": "new building",
37
+ "A1": "major alteration (use/occupancy)",
38
+ "A2": "minor alteration",
39
+ "A3": "minor work / interior",
40
+ "DM": "demolition",
41
+ "SG": "sign",
42
+ "PL": "plumbing",
43
+ "EQ": "equipment",
44
+ }
45
+
46
+ # Default filter: focus on "what are they building" β€” new construction,
47
+ # major alterations, demolitions. Skip minor mechanical permits.
48
+ DEFAULT_JOB_TYPES = ("NB", "A1", "DM")
49
+
50
+
51
+ @dataclass
52
+ class Permit:
53
+ job_id: str
54
+ job_type: str
55
+ job_type_label: str
56
+ permit_status: str
57
+ issuance_date: str
58
+ expiration_date: str | None
59
+ address: str
60
+ borough: str
61
+ bbl: str | None
62
+ lat: float
63
+ lon: float
64
+ owner_business: str | None
65
+ permittee_business: str | None
66
+ nta_name: str | None
67
+
68
+
69
+ def permits_in_bbox(min_lat: float, min_lon: float,
70
+ max_lat: float, max_lon: float,
71
+ job_types: tuple[str, ...] = DEFAULT_JOB_TYPES,
72
+ since: date | None = None,
73
+ limit: int = 5000) -> list[Permit]:
74
+ """Pull DOB permits intersecting a bounding box, recently issued, with
75
+ matching job types. We expand from polygon to bbox and rely on the
76
+ caller to do exact point-in-polygon filtering."""
77
+ if since is None:
78
+ since = date.today() - timedelta(days=540) # ~18 months
79
+ # gis_latitude/gis_longitude are stored as text in this dataset; cast
80
+ # to number for the bbox compare. issuance_date is a floating timestamp
81
+ # surfaced as 'MM/DD/YYYY' string β€” cast explicitly to floating_timestamp
82
+ # so the comparator parses ISO dates correctly. BETWEEN is picky on text
83
+ # columns, so use explicit >= / <= operators.
84
+ where = (
85
+ f"job_type IN ({','.join(repr(t) for t in job_types)})"
86
+ f" AND issuance_date::floating_timestamp >= '{since.isoformat()}'"
87
+ f" AND gis_latitude::number >= {min_lat}"
88
+ f" AND gis_latitude::number <= {max_lat}"
89
+ f" AND gis_longitude::number >= {min_lon}"
90
+ f" AND gis_longitude::number <= {max_lon}"
91
+ )
92
+ r = httpx.get(URL, params={
93
+ "$select": ",".join([
94
+ "job__", "job_type", "permit_status", "issuance_date",
95
+ "expiration_date", "house__", "street_name", "borough",
96
+ "block", "lot",
97
+ "gis_latitude", "gis_longitude", "owner_s_business_name",
98
+ "permittee_s_business_name", "gis_nta_name",
99
+ ]),
100
+ "$where": where,
101
+ "$order": "issuance_date desc",
102
+ "$limit": str(limit),
103
+ }, timeout=60)
104
+ r.raise_for_status()
105
+ out: list[Permit] = []
106
+ for row in r.json():
107
+ try:
108
+ lat = float(row["gis_latitude"])
109
+ lon = float(row["gis_longitude"])
110
+ except (KeyError, ValueError, TypeError):
111
+ continue
112
+ addr = " ".join(filter(None, [
113
+ row.get("house__"),
114
+ (row.get("street_name") or "").title(),
115
+ ])).strip()
116
+ # DOB has no `bbl` column; compose from borough + block + lot.
117
+ # Borough codes: MAN=1, BX=2, BK=3, QN=4, SI=5.
118
+ boro_code = {"MANHATTAN": "1", "BRONX": "2", "BROOKLYN": "3",
119
+ "QUEENS": "4", "STATEN ISLAND": "5"}.get(
120
+ (row.get("borough") or "").upper())
121
+ block = (row.get("block") or "").lstrip("0")
122
+ lot = (row.get("lot") or "").lstrip("0")
123
+ bbl = (f"{boro_code}-{block.zfill(5)}-{lot.zfill(4)}"
124
+ if boro_code and block and lot else None)
125
+ out.append(Permit(
126
+ job_id=row.get("job__", ""),
127
+ job_type=row.get("job_type", ""),
128
+ job_type_label=JOB_TYPE_LABELS.get(row.get("job_type", ""), row.get("job_type", "")),
129
+ permit_status=row.get("permit_status", ""),
130
+ issuance_date=(row.get("issuance_date") or "")[:10],
131
+ expiration_date=(row.get("expiration_date") or "")[:10] or None,
132
+ address=addr,
133
+ borough=(row.get("borough") or "").title(),
134
+ bbl=bbl,
135
+ lat=lat,
136
+ lon=lon,
137
+ owner_business=row.get("owner_s_business_name"),
138
+ permittee_business=row.get("permittee_s_business_name"),
139
+ nta_name=row.get("gis_nta_name"),
140
+ ))
141
+ return out
142
+
143
+
144
+ def permits_in_polygon(polygon, polygon_crs: str = "EPSG:4326",
145
+ job_types: tuple[str, ...] = DEFAULT_JOB_TYPES,
146
+ since: date | None = None) -> list[Permit]:
147
+ """Permits inside a polygon. Uses bbox prefilter + shapely contains."""
148
+ g = gpd.GeoDataFrame(geometry=[polygon], crs=polygon_crs).to_crs("EPSG:4326")
149
+ geom = g.iloc[0].geometry
150
+ minx, miny, maxx, maxy = geom.bounds
151
+ raw = permits_in_bbox(miny, minx, maxy, maxx, job_types=job_types, since=since)
152
+ out: list[Permit] = []
153
+ for p in raw:
154
+ pt = Point(p.lon, p.lat)
155
+ if geom.contains(pt) or geom.intersects(pt):
156
+ out.append(p)
157
+ # Dedupe by job_id (one job can have multiple permits as work proceeds)
158
+ seen: dict[str, Permit] = {}
159
+ for p in out:
160
+ # Keep the most-recently-issued permit per job
161
+ cur = seen.get(p.job_id)
162
+ if cur is None or (p.issuance_date or "") > (cur.issuance_date or ""):
163
+ seen[p.job_id] = p
164
+ return list(seen.values())
165
+
166
+
167
+ def cross_reference_flood(permits: list[Permit]) -> list[dict[str, Any]]:
168
+ """Tag each permit with which flood layers cover its point.
169
+ Adds: in_sandy (bool), dep_class (highest depth class hit across DEP scenarios),
170
+ dep_scenarios (list of scenario ids that fired)."""
171
+ if not permits:
172
+ return []
173
+ from app.flood_layers import dep_stormwater, sandy_inundation
174
+ pts = gpd.GeoDataFrame(
175
+ geometry=[Point(p.lon, p.lat) for p in permits],
176
+ crs="EPSG:4326",
177
+ ).to_crs("EPSG:2263")
178
+ pts["_pid"] = list(range(len(pts)))
179
+
180
+ sandy_flags = sandy_inundation.join(pts).reset_index(drop=True).tolist()
181
+
182
+ dep_hits = {scen: dep_stormwater.join(pts, scen)["depth_class"].astype(int).tolist()
183
+ for scen in ("dep_extreme_2080", "dep_moderate_2050", "dep_moderate_current")}
184
+
185
+ out = []
186
+ for i, p in enumerate(permits):
187
+ scen_hits = {s: dep_hits[s][i] for s in dep_hits}
188
+ max_class = max(scen_hits.values(), default=0)
189
+ active_scens = [s for s, c in scen_hits.items() if c > 0]
190
+ out.append({
191
+ **asdict(p),
192
+ "in_sandy": bool(sandy_flags[i]),
193
+ "dep_max_class": max_class,
194
+ "dep_scenarios": active_scens,
195
+ "any_flood_layer_hit": bool(sandy_flags[i] or max_class > 0),
196
+ })
197
+ return out
198
+
199
+
200
+ def summary_for_polygon(polygon, polygon_crs: str = "EPSG:4326",
201
+ since_days: int = 540,
202
+ top_n: int = 8) -> dict:
203
+ """Full polygon-mode summary: list active permits, cross-reference each
204
+ with flood layers, return aggregate counts + a top-N projects-of-concern
205
+ list (those that hit at least one flood layer, ranked by max DEP class
206
+ + Sandy hit)."""
207
+ since = date.today() - timedelta(days=since_days)
208
+ permits = permits_in_polygon(polygon, polygon_crs=polygon_crs, since=since)
209
+ enriched = cross_reference_flood(permits)
210
+
211
+ by_type: Counter = Counter(e["job_type_label"] for e in enriched)
212
+ by_status: Counter = Counter(e["permit_status"] for e in enriched)
213
+ n_total = len(enriched)
214
+ n_sandy = sum(1 for e in enriched if e["in_sandy"])
215
+ n_dep_any = sum(1 for e in enriched if e["dep_max_class"] > 0)
216
+ n_dep_severe = sum(1 for e in enriched if e["dep_max_class"] >= 2)
217
+ n_any_flood = sum(1 for e in enriched if e["any_flood_layer_hit"])
218
+
219
+ # Rank: severity = (in_sandy * 3) + dep_max_class
220
+ def severity(e):
221
+ return (3 if e["in_sandy"] else 0) + e["dep_max_class"]
222
+ flagged = sorted(
223
+ [e for e in enriched if e["any_flood_layer_hit"]],
224
+ key=severity, reverse=True,
225
+ )[:top_n]
226
+
227
+ # Light projection of every permit for map pinning (no need to ship the
228
+ # full permit record for the not-flagged ones β€” the map only needs lat,
229
+ # lon, address, job_type_label, and the flood-flag fields).
230
+ all_pins = [
231
+ {
232
+ "lat": e["lat"],
233
+ "lon": e["lon"],
234
+ "address": e["address"],
235
+ "job_type": e["job_type"],
236
+ "in_sandy": e["in_sandy"],
237
+ "dep_max_class": e["dep_max_class"],
238
+ "any_flood": e["any_flood_layer_hit"],
239
+ }
240
+ for e in enriched
241
+ ]
242
+ return {
243
+ "since": since.isoformat(),
244
+ "n_total": n_total,
245
+ "n_in_sandy": n_sandy,
246
+ "n_in_dep_any": n_dep_any,
247
+ "n_in_dep_severe": n_dep_severe,
248
+ "n_any_flood": n_any_flood,
249
+ "by_job_type": dict(by_type.most_common()),
250
+ "by_permit_status":dict(by_status.most_common()),
251
+ "flagged_top": flagged,
252
+ "all_pins": all_pins,
253
+ "all_count": n_total,
254
+ }
255
+
256
+
257
+ def now_iso() -> str:
258
+ return datetime.utcnow().date().isoformat()
app/context/eo_chip_cache.py ADDED
@@ -0,0 +1,345 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Per-query EO chip cache β€” Sentinel-2 L2A, Sentinel-1 RTC, DEM.
2
+
3
+ Fetches a co-registered (S2L2A, S1RTC, DEM) chip centered on (lat, lon)
4
+ and returns a dict of torch tensors ready for TerraMind-NYC inference.
5
+ The TerraMind base was trained with `temporal_n_timestamps=4`, so this
6
+ helper expands a single S2/S1 acquisition to T=4 by repetition along
7
+ the temporal axis. Single-timestep nowcasting trades some training-
8
+ distribution match for a much simpler runtime β€” the published LoRA
9
+ adapters still produce sensible argmax masks at T=1 / tiled.
10
+
11
+ Failure semantics mirror prithvi_live: every dependency or network
12
+ failure is converted to a clean `{ok: False, skipped: <reason>}`
13
+ result, never a raised exception. Callers (FSM specialists) that
14
+ chain off the chip can short-circuit on `ok=False` and skip the
15
+ specialist instead of surfacing a noisy error.
16
+ """
17
+ from __future__ import annotations
18
+
19
+ import concurrent.futures
20
+ import logging
21
+ import os
22
+ import threading
23
+ import time
24
+ from typing import Any
25
+
26
+ log = logging.getLogger("riprap.eo_chip_cache")
27
+
28
+ ENABLE = os.environ.get("RIPRAP_EO_CHIP_ENABLE", "1").lower() in ("1", "true", "yes")
29
+ SEARCH_DAYS = int(os.environ.get("RIPRAP_EO_CHIP_SEARCH_DAYS", "120"))
30
+ MAX_CLOUD_PCT = float(os.environ.get("RIPRAP_EO_CHIP_MAX_CLOUD", "30"))
31
+ CHIP_PX = int(os.environ.get("RIPRAP_EO_CHIP_PX", "224"))
32
+ PIXEL_M = 10
33
+ N_TIMESTEPS = 4
34
+
35
+ # 12-band S2 L2A in TerraMind's expected order.
36
+ S2_BANDS = ["B01", "B02", "B03", "B04", "B05", "B06", "B07",
37
+ "B08", "B8A", "B09", "B11", "B12"]
38
+
39
+ # Sentinel-1 RTC on Planetary Computer publishes vv/vh polarisations.
40
+ S1_BANDS = ["vv", "vh"]
41
+
42
+
43
+ def _has_required_deps() -> tuple[bool, str | None]:
44
+ missing: list[str] = []
45
+ for name in ("planetary_computer", "pystac_client",
46
+ "rioxarray", "xarray", "torch", "numpy"):
47
+ try:
48
+ __import__(name)
49
+ except ImportError:
50
+ missing.append(name)
51
+ if missing:
52
+ return False, ", ".join(missing)
53
+ return True, None
54
+
55
+
56
+ _DEPS_OK, _DEPS_MISSING = _has_required_deps()
57
+ _FETCH_LOCK = threading.Lock()
58
+
59
+
60
+ def _search_s2(lat: float, lon: float):
61
+ """Return (item, cloud_cover) for the most recent low-cloud S2L2A
62
+ acquisition near (lat, lon), or (None, None) if no scene exists."""
63
+ import datetime as dt
64
+
65
+ import planetary_computer as pc
66
+ from pystac_client import Client
67
+ end = dt.datetime.utcnow().date()
68
+ start = end - dt.timedelta(days=SEARCH_DAYS)
69
+ client = Client.open(
70
+ "https://planetarycomputer.microsoft.com/api/stac/v1",
71
+ modifier=pc.sign_inplace,
72
+ )
73
+ delta = 0.02
74
+ search = client.search(
75
+ collections=["sentinel-2-l2a"],
76
+ bbox=[lon - delta, lat - delta, lon + delta, lat + delta],
77
+ datetime=f"{start}/{end}",
78
+ query={"eo:cloud_cover": {"lt": MAX_CLOUD_PCT}},
79
+ max_items=20,
80
+ )
81
+ items = sorted(
82
+ search.items(),
83
+ key=lambda it: (it.properties.get("eo:cloud_cover", 100),
84
+ -(it.datetime.timestamp() if it.datetime else 0)),
85
+ )
86
+ if not items:
87
+ return None, None
88
+ item = items[0]
89
+ cc = float(item.properties.get("eo:cloud_cover", -1))
90
+ return item, cc
91
+
92
+
93
+ def _search_s1(item_dt, lat: float, lon: float):
94
+ """Return the closest Sentinel-1 RTC acquisition to the given S2
95
+ datetime, or None if Planetary Computer has nothing nearby."""
96
+ import datetime as dt
97
+
98
+ import planetary_computer as pc
99
+ from pystac_client import Client
100
+ win = dt.timedelta(days=10)
101
+ start = item_dt - win
102
+ end = item_dt + win
103
+ client = Client.open(
104
+ "https://planetarycomputer.microsoft.com/api/stac/v1",
105
+ modifier=pc.sign_inplace,
106
+ )
107
+ delta = 0.02
108
+ search = client.search(
109
+ collections=["sentinel-1-rtc"],
110
+ bbox=[lon - delta, lat - delta, lon + delta, lat + delta],
111
+ datetime=f"{start.isoformat()}/{end.isoformat()}",
112
+ max_items=10,
113
+ )
114
+ items = list(search.items())
115
+ if not items:
116
+ return None
117
+ items.sort(key=lambda it:
118
+ abs((it.datetime - item_dt).total_seconds())
119
+ if it.datetime else 1e18)
120
+ return items[0]
121
+
122
+
123
+ def _read_band(href, bbox_xy_meters, epsg):
124
+ """Read a single COG band, clipped to the bbox, and resample to
125
+ CHIP_PX Γ— CHIP_PX. Returns a numpy array (CHIP_PX, CHIP_PX) float32.
126
+ """
127
+ import numpy as np
128
+ import rioxarray # noqa: F401
129
+ da = rioxarray.open_rasterio(href, masked=False).squeeze(drop=True)
130
+ da = da.rio.clip_box(minx=bbox_xy_meters[0], miny=bbox_xy_meters[1],
131
+ maxx=bbox_xy_meters[2], maxy=bbox_xy_meters[3])
132
+ if da.shape[-2] != CHIP_PX or da.shape[-1] != CHIP_PX:
133
+ # Resample (nearest is fine for the 10/20/60 m S2 mix; S1 is 10 m,
134
+ # DEM is 30 m and benefits from bilinear; we keep nearest for
135
+ # simplicity β€” the TerraMind LoRA was trained against terratorch's
136
+ # default resampler which is also nearest).
137
+ da = da.rio.reproject(
138
+ f"EPSG:{epsg}", shape=(CHIP_PX, CHIP_PX), resampling=0
139
+ )
140
+ arr = da.values.astype("float32")
141
+ return np.nan_to_num(arr)
142
+
143
+
144
+ def _fetch_modalities(lat: float, lon: float, timeout_s: float = 60.0) -> dict[str, Any]:
145
+ """Fetch S2L2A + S1RTC + DEM as numpy arrays, resampled to a common
146
+ CHIP_PX Γ— CHIP_PX grid centered on (lat, lon).
147
+ """
148
+ import numpy as np
149
+ from pyproj import Transformer
150
+
151
+ t0 = time.time()
152
+ item, cc = _search_s2(lat, lon)
153
+ if item is None:
154
+ return {"ok": False,
155
+ "skipped": f"no <{MAX_CLOUD_PCT}% cloud S2 in last "
156
+ f"{SEARCH_DAYS}d"}
157
+ if "proj:epsg" in item.properties:
158
+ epsg = int(item.properties["proj:epsg"])
159
+ else:
160
+ code = item.properties.get("proj:code", "")
161
+ if not code.startswith("EPSG:"):
162
+ return {"ok": False,
163
+ "skipped": "STAC item missing proj:epsg / proj:code"}
164
+ epsg = int(code.split(":", 1)[1])
165
+
166
+ fwd = Transformer.from_crs("EPSG:4326", f"EPSG:{epsg}", always_xy=True)
167
+ cx, cy = fwd.transform(lon, lat)
168
+ half_m = CHIP_PX / 2 * PIXEL_M
169
+ bbox = (cx - half_m, cy - half_m, cx + half_m, cy + half_m)
170
+
171
+ if time.time() - t0 > timeout_s:
172
+ return {"ok": False, "skipped": "STAC search exceeded budget"}
173
+
174
+ # ---- S2L2A: 12 bands ------------------------------------------------
175
+ s2_arrs = []
176
+ try:
177
+ for b in S2_BANDS:
178
+ href = item.assets[b].href
179
+ s2_arrs.append(_read_band(href, bbox, epsg))
180
+ except Exception as e:
181
+ log.warning("eo_chip: S2 band fetch failed (%s); aborting", e)
182
+ return {"ok": False, "err": f"S2 fetch failed: {type(e).__name__}: {e}"}
183
+ s2 = np.stack(s2_arrs) # (12, H, W)
184
+ if s2.mean() > 1.0:
185
+ s2 = s2 / 10000.0 # scale L2A reflectance from int16 to ~[0, 1]
186
+
187
+ # ---- S1RTC: 2 polarisations (best effort) ---------------------------
188
+ s1: np.ndarray | None = None
189
+ s1_meta: dict[str, Any] = {}
190
+ if time.time() - t0 < timeout_s:
191
+ try:
192
+ s1_item = _search_s1(item.datetime, lat, lon)
193
+ if s1_item is not None:
194
+ s1_arrs = []
195
+ for b in S1_BANDS:
196
+ href = s1_item.assets[b].href
197
+ s1_arrs.append(_read_band(href, bbox, epsg))
198
+ s1 = np.stack(s1_arrs)
199
+ s1_meta = {
200
+ "scene_id": s1_item.id,
201
+ "datetime": (s1_item.datetime.isoformat()
202
+ if s1_item.datetime else None),
203
+ }
204
+ except Exception as e:
205
+ log.warning("eo_chip: S1 fetch best-effort failed: %s", e)
206
+
207
+ # ---- DEM: Copernicus 30 m via planetary_computer (best effort) ------
208
+ dem: np.ndarray | None = None
209
+ if time.time() - t0 < timeout_s:
210
+ try:
211
+ import planetary_computer as pc
212
+ from pystac_client import Client
213
+ client = Client.open(
214
+ "https://planetarycomputer.microsoft.com/api/stac/v1",
215
+ modifier=pc.sign_inplace,
216
+ )
217
+ dem_search = client.search(
218
+ collections=["cop-dem-glo-30"],
219
+ bbox=[lon - 0.02, lat - 0.02, lon + 0.02, lat + 0.02],
220
+ max_items=1,
221
+ )
222
+ dem_items = list(dem_search.items())
223
+ if dem_items:
224
+ href = dem_items[0].assets["data"].href
225
+ dem = _read_band(href, bbox, epsg)
226
+ dem = dem[None, :, :] # add channel dim
227
+ except Exception as e:
228
+ log.warning("eo_chip: DEM fetch best-effort failed: %s", e)
229
+
230
+ return {
231
+ "ok": True,
232
+ "lat": lat, "lon": lon,
233
+ "epsg": epsg, "chip_px": CHIP_PX, "pixel_m": PIXEL_M,
234
+ "s2": s2, "s1": s1, "dem": dem,
235
+ "s2_meta": {
236
+ "scene_id": item.id,
237
+ "datetime": (item.datetime.isoformat() if item.datetime else None),
238
+ "cloud_cover": cc,
239
+ },
240
+ "s1_meta": s1_meta,
241
+ "elapsed_s": round(time.time() - t0, 2),
242
+ }
243
+
244
+
245
+ def _to_terramind_tensors(modalities: dict[str, Any]) -> dict[str, Any]:
246
+ """Shape numpy modality arrays into the (B, C, T, H, W) tensors
247
+ TerraMind expects with `temporal_n_timestamps=4`. Single-timestep
248
+ fetches get tiled to T=4 β€” same observation in every slot.
249
+ """
250
+ import torch
251
+ s2 = modalities["s2"] # (12, H, W)
252
+ s2_t = torch.from_numpy(s2).float().unsqueeze(1) # (12, 1, H, W)
253
+ s2_t = s2_t.repeat(1, N_TIMESTEPS, 1, 1).unsqueeze(0) # (1, 12, T, H, W)
254
+ chips = {"S2L2A": s2_t}
255
+ if modalities.get("s1") is not None:
256
+ s1 = modalities["s1"] # (2, H, W)
257
+ s1_t = torch.from_numpy(s1).float().unsqueeze(1)
258
+ s1_t = s1_t.repeat(1, N_TIMESTEPS, 1, 1).unsqueeze(0)
259
+ chips["S1RTC"] = s1_t
260
+ if modalities.get("dem") is not None:
261
+ dem = modalities["dem"] # (1, H, W)
262
+ dem_t = torch.from_numpy(dem).float().unsqueeze(1)
263
+ dem_t = dem_t.repeat(1, N_TIMESTEPS, 1, 1).unsqueeze(0)
264
+ chips["DEM"] = dem_t
265
+ return chips
266
+
267
+
268
+ def _fetch_and_build(lat: float, lon: float, timeout_s: float) -> dict[str, Any]:
269
+ """Inner fetch + tensor build, run inside a bounded thread."""
270
+ with _FETCH_LOCK:
271
+ try:
272
+ modalities = _fetch_modalities(lat, lon, timeout_s=timeout_s)
273
+ except Exception as e:
274
+ log.exception("eo_chip: fetch failed")
275
+ return {"ok": False, "err": f"{type(e).__name__}: {e}"}
276
+ if not modalities.get("ok"):
277
+ return modalities
278
+ try:
279
+ modalities["tensors"] = _to_terramind_tensors(modalities)
280
+ except Exception as e:
281
+ log.exception("eo_chip: tensor build failed")
282
+ return {"ok": False,
283
+ "err": f"tensor build failed: {type(e).__name__}: {e}"}
284
+ # Compute the chip's WGS84 bbox so downstream TerraMind specialists
285
+ # can polygonise their predictions onto the map. The chip is
286
+ # CHIP_PX Γ— CHIP_PX at PIXEL_M (10 m) in the scene's UTM zone;
287
+ # reproject the four corners to EPSG:4326 and use the
288
+ # axis-aligned envelope.
289
+ try:
290
+ from pyproj import Transformer
291
+ half_m = (CHIP_PX * PIXEL_M) / 2.0
292
+ t_to_utm = Transformer.from_crs(
293
+ "EPSG:4326", f"EPSG:{modalities['epsg']}", always_xy=True)
294
+ t_to_4326 = Transformer.from_crs(
295
+ f"EPSG:{modalities['epsg']}", "EPSG:4326", always_xy=True)
296
+ cx, cy = t_to_utm.transform(lon, lat)
297
+ corners_utm = [
298
+ (cx - half_m, cy - half_m),
299
+ (cx - half_m, cy + half_m),
300
+ (cx + half_m, cy - half_m),
301
+ (cx + half_m, cy + half_m),
302
+ ]
303
+ corners_ll = [t_to_4326.transform(x, y) for x, y in corners_utm]
304
+ lons = [c[0] for c in corners_ll]
305
+ lats = [c[1] for c in corners_ll]
306
+ modalities["bounds_4326"] = (
307
+ min(lons), min(lats), max(lons), max(lats))
308
+ except Exception:
309
+ log.exception("eo_chip: bounds_4326 reprojection failed")
310
+ return modalities
311
+
312
+
313
+ def fetch(lat: float, lon: float, timeout_s: float = 60.0) -> dict[str, Any]:
314
+ """Run the chip pipeline. Always returns a dict with at minimum
315
+ `{ok, skipped|err, ...}`; on success the dict carries the
316
+ co-registered numpy arrays plus `tensors` (the TerraMind-shaped
317
+ torch dict).
318
+
319
+ Runs in a daemon thread so that STAC searches and COG band downloads
320
+ (which use requests/rioxarray without per-call timeouts) are bounded
321
+ by a hard wall-clock deadline even when the network hangs.
322
+ """
323
+ if not ENABLE:
324
+ return {"ok": False, "skipped": "RIPRAP_EO_CHIP_ENABLE=0"}
325
+ if not _DEPS_OK:
326
+ return {"ok": False,
327
+ "skipped": f"deps unavailable on this deployment: "
328
+ f"{_DEPS_MISSING}"}
329
+ # Hard wall-clock cap: pystac_client / rioxarray COG reads don't expose
330
+ # uniform per-request timeouts, so we bound the whole pipeline here.
331
+ hard_timeout = timeout_s + 15.0
332
+ # Propagate the parent thread's emissions tracker into the worker so
333
+ # any inference._post calls made inside _fetch_and_build are recorded.
334
+ from app import emissions as _emissions
335
+ _parent_tracker = _emissions.current()
336
+ with concurrent.futures.ThreadPoolExecutor(
337
+ max_workers=1,
338
+ initializer=lambda t=_parent_tracker: _emissions.install(t),
339
+ ) as pool:
340
+ future = pool.submit(_fetch_and_build, lat, lon, timeout_s)
341
+ try:
342
+ return future.result(timeout=hard_timeout)
343
+ except concurrent.futures.TimeoutError:
344
+ log.warning("eo_chip: hard timeout after %.0fs (STAC/COG hung)", hard_timeout)
345
+ return {"ok": False, "skipped": f"eo_chip timed out after {hard_timeout:.0f}s"}
app/context/floodnet.py ADDED
@@ -0,0 +1,148 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FloodNet NYC β€” live ultrasonic flood sensor network.
2
+
3
+ Hasura GraphQL endpoint, no auth, ~350 sensors. Used for:
4
+ - sensors_near(lat, lon, radius_m) β†’ list of deployments
5
+ - flood_events_for(deployment_ids, since) β†’ labeled flood events per sensor
6
+ """
7
+ from __future__ import annotations
8
+
9
+ from dataclasses import dataclass
10
+ from datetime import datetime, timedelta, timezone
11
+ from typing import Any
12
+
13
+ import httpx
14
+
15
+ URL = "https://api.floodnet.nyc/v1/graphql"
16
+ DOC_ID = "floodnet"
17
+ CITATION = "FloodNet NYC ultrasonic depth sensors (api.floodnet.nyc)"
18
+
19
+
20
+ @dataclass
21
+ class Sensor:
22
+ deployment_id: str
23
+ name: str
24
+ street: str
25
+ borough: str
26
+ status: str
27
+ deployed_at: str | None
28
+ lat: float | None = None
29
+ lon: float | None = None
30
+
31
+
32
+ @dataclass
33
+ class FloodEvent:
34
+ deployment_id: str
35
+ start_time: str
36
+ end_time: str | None
37
+ max_depth_mm: int | None
38
+ label: str | None
39
+
40
+
41
+ def _gql(query: str, variables: dict[str, Any]) -> dict:
42
+ r = httpx.post(URL, json={"query": query, "variables": variables},
43
+ timeout=20, verify=False)
44
+ r.raise_for_status()
45
+ j = r.json()
46
+ if "errors" in j:
47
+ raise RuntimeError(f"FloodNet GraphQL error: {j['errors']}")
48
+ return j["data"]
49
+
50
+
51
+ _NEAR_Q = """
52
+ query Near($lat: Float!, $lon: Float!, $r: Float!) {
53
+ deployments_within_radius(args:{lat:$lat, lon:$lon, radius_meters:$r},
54
+ order_by:{date_deployed: asc}) {
55
+ deployment_id
56
+ name
57
+ sensor_address_street
58
+ sensor_address_borough
59
+ sensor_status
60
+ date_deployed
61
+ location
62
+ }
63
+ }"""
64
+
65
+
66
+ def _parse_location(loc) -> tuple[float | None, float | None]:
67
+ """Hasura PostGIS geometry returned as a GeoJSON object."""
68
+ if not loc or not isinstance(loc, dict):
69
+ return None, None
70
+ coords = loc.get("coordinates")
71
+ if not coords or len(coords) < 2:
72
+ return None, None
73
+ return coords[1], coords[0] # (lat, lon) from (lon, lat)
74
+
75
+
76
+ def sensors_near(lat: float, lon: float, radius_m: float = 1000) -> list[Sensor]:
77
+ d = _gql(_NEAR_Q, {"lat": lat, "lon": lon, "r": radius_m})
78
+ out = []
79
+ for row in d["deployments_within_radius"]:
80
+ slat, slon = _parse_location(row.get("location"))
81
+ out.append(Sensor(
82
+ deployment_id=row["deployment_id"],
83
+ name=row["name"] or "",
84
+ street=row.get("sensor_address_street") or "",
85
+ borough=row.get("sensor_address_borough") or "",
86
+ status=row.get("sensor_status") or "",
87
+ deployed_at=row.get("date_deployed"),
88
+ lat=slat,
89
+ lon=slon,
90
+ ))
91
+ return out
92
+
93
+
94
+ _EVENTS_Q = """
95
+ query Events($ids: [String!], $since: timestamp!) {
96
+ sensor_events(where:{
97
+ deployment_id:{_in:$ids},
98
+ start_time:{_gte:$since},
99
+ label:{_eq:"flood"}
100
+ }, order_by:{start_time: desc}, limit: 200) {
101
+ deployment_id
102
+ start_time
103
+ end_time
104
+ max_depth_proc_mm
105
+ label
106
+ }
107
+ }"""
108
+
109
+
110
+ def flood_events_for(deployment_ids: list[str],
111
+ since: datetime | None = None) -> list[FloodEvent]:
112
+ if not deployment_ids:
113
+ return []
114
+ if since is None:
115
+ since = datetime.now(timezone.utc) - timedelta(days=365 * 3)
116
+ d = _gql(_EVENTS_Q, {
117
+ "ids": deployment_ids,
118
+ "since": since.isoformat(timespec="seconds").replace("+00:00", ""),
119
+ })
120
+ return [
121
+ FloodEvent(
122
+ deployment_id=row["deployment_id"],
123
+ start_time=row["start_time"],
124
+ end_time=row.get("end_time"),
125
+ max_depth_mm=row.get("max_depth_proc_mm"),
126
+ label=row.get("label"),
127
+ )
128
+ for row in d["sensor_events"]
129
+ ]
130
+
131
+
132
+ def summary_for_point(lat: float, lon: float, radius_m: float = 600) -> dict:
133
+ """One-shot summary used by the FSM node and the cited paragraph."""
134
+ sensors = sensors_near(lat, lon, radius_m)
135
+ ids = [s.deployment_id for s in sensors]
136
+ events = flood_events_for(ids)
137
+ by_dep: dict[str, list[FloodEvent]] = {}
138
+ for e in events:
139
+ by_dep.setdefault(e.deployment_id, []).append(e)
140
+ peak = max((e for e in events if e.max_depth_mm is not None),
141
+ key=lambda e: e.max_depth_mm or 0, default=None)
142
+ return {
143
+ "n_sensors": len(sensors),
144
+ "sensors": [vars(s) for s in sensors],
145
+ "n_flood_events_3y": len(events),
146
+ "n_sensors_with_events": len(by_dep),
147
+ "peak_event": vars(peak) if peak else None,
148
+ }
app/context/gliner_extract.py ADDED
@@ -0,0 +1,147 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """GLiNER (urchade/gliner_medium-v2.1) typed-entity extraction over the
2
+ RAG retriever's top paragraphs.
3
+
4
+ Adds structured fields to the reconciler's grounding context. For each
5
+ RAG chunk the specialist emits, GLiNER produces a list of typed spans
6
+ with one of five labels:
7
+
8
+ nyc_location (e.g. "Coney Island")
9
+ dollar_amount (e.g. "$5.6 million")
10
+ date_range (e.g. "fiscal year 2025-2027")
11
+ agency (e.g. "NYC DEP")
12
+ infrastructure_project (e.g. "Bluebelt expansion")
13
+
14
+ The doc_id for emission is `gliner_<source>` where `<source>` is the
15
+ RAG chunk's doc_id stripped of its `rag_` prefix. So `rag_comptroller`
16
+ becomes `gliner_comptroller`. The reconciler can then cite typed
17
+ fields with `[gliner_comptroller]`.
18
+
19
+ License: Apache-2.0 β€” `urchade/gliner_medium-v2.1` (NOT the
20
+ `gliner_base` variant, which is CC-BY-NC-4.0). See
21
+ experiments/shared/licenses.md.
22
+ """
23
+
24
+ from __future__ import annotations
25
+
26
+ import logging
27
+ import os
28
+ from dataclasses import dataclass
29
+
30
+ log = logging.getLogger("riprap.gliner")
31
+
32
+ ENTITY_LABELS = [
33
+ "nyc_location",
34
+ "dollar_amount",
35
+ "date_range",
36
+ "agency",
37
+ "infrastructure_project",
38
+ ]
39
+
40
+ DEFAULT_THRESHOLD = float(os.environ.get("RIPRAP_GLINER_THRESHOLD", "0.45"))
41
+ MODEL_NAME = os.environ.get("RIPRAP_GLINER_MODEL", "urchade/gliner_medium-v2.1")
42
+ ENABLE = os.environ.get("RIPRAP_GLINER_ENABLE", "1").lower() in ("1", "true", "yes")
43
+
44
+ _MODEL = None # lazy
45
+
46
+
47
+ @dataclass
48
+ class Extraction:
49
+ label: str
50
+ text: str
51
+ score: float
52
+
53
+
54
+ def _ensure_model():
55
+ """Lazy GLiNER load. Returns None if disabled or load fails so
56
+ callers can silently fall back to no-op."""
57
+ global _MODEL
58
+ if not ENABLE:
59
+ return None
60
+ if _MODEL is not None:
61
+ return _MODEL
62
+ try:
63
+ from gliner import GLiNER
64
+ log.info("gliner: loading %s", MODEL_NAME)
65
+ _MODEL = GLiNER.from_pretrained(MODEL_NAME)
66
+ except Exception:
67
+ log.exception("gliner: load failed; specialist will no-op")
68
+ _MODEL = False # sentinel
69
+ return _MODEL or None
70
+
71
+
72
+ def warm():
73
+ _ensure_model()
74
+
75
+
76
+ def _source_short(rag_doc_id: str) -> str:
77
+ """`rag_comptroller` -> `comptroller`. Anything not prefixed `rag_`
78
+ passes through unchanged."""
79
+ return rag_doc_id[4:] if rag_doc_id.startswith("rag_") else rag_doc_id
80
+
81
+
82
+ def extract_for_chunk(text: str, threshold: float = DEFAULT_THRESHOLD) -> list[Extraction]:
83
+ if not text:
84
+ return []
85
+
86
+ # v0.4.5 β€” try the MI300X service first. The remote handles its
87
+ # own GLiNER load; this lets cpu-basic surfaces run typed
88
+ # extraction without baking gliner into the image.
89
+ try:
90
+ from app import inference as _inf
91
+ if _inf.remote_enabled():
92
+ remote = _inf.gliner_extract(text, ENTITY_LABELS)
93
+ if remote.get("ok"):
94
+ return [
95
+ Extraction(label=e["label"], text=e["text"],
96
+ score=float(e.get("score", 0)))
97
+ for e in remote.get("entities", [])
98
+ if e.get("score", 0) >= threshold
99
+ ]
100
+ except _inf.RemoteUnreachable as e:
101
+ log.info("gliner: remote unreachable (%s); local fallback", e)
102
+ except Exception:
103
+ log.exception("gliner: remote call failed; local fallback")
104
+
105
+ model = _ensure_model()
106
+ if model is None:
107
+ return []
108
+ raw = model.predict_entities(text, ENTITY_LABELS, threshold=threshold)
109
+ return [Extraction(label=r["label"], text=r["text"],
110
+ score=float(r["score"])) for r in raw]
111
+
112
+
113
+ def extract_for_rag_hits(hits: list[dict],
114
+ threshold: float = DEFAULT_THRESHOLD,
115
+ max_hits: int = 3) -> dict[str, dict]:
116
+ """Run GLiNER on the top-`max_hits` RAG hits. Returns a dict keyed by
117
+ short source id (e.g. "comptroller") with the structured payload
118
+ that the FSM stores into state["gliner"] and that
119
+ reconcile.build_documents() consumes."""
120
+ out: dict[str, dict] = {}
121
+ if not hits:
122
+ return out
123
+ for h in hits[:max_hits]:
124
+ source = _source_short(h.get("doc_id", "rag_unknown"))
125
+ ents = extract_for_chunk(h.get("text", ""), threshold=threshold)
126
+ if not ents:
127
+ continue
128
+ # Dedup verbatim repeats (common in agency PDFs that repeat
129
+ # "DEP" 13 times in a methodology section).
130
+ seen = set()
131
+ deduped: list[Extraction] = []
132
+ for e in ents:
133
+ key = (e.label, e.text.lower())
134
+ if key in seen:
135
+ continue
136
+ seen.add(key)
137
+ deduped.append(e)
138
+ out[source] = {
139
+ "rag_doc_id": h.get("doc_id"),
140
+ "title": h.get("title"),
141
+ "paragraph_excerpt": h.get("text", "")[:240]
142
+ + ("…" if len(h.get("text", "")) > 240 else ""),
143
+ "n_entities": len(deduped),
144
+ "entities": [{"label": e.label, "text": e.text,
145
+ "score": round(e.score, 3)} for e in deduped],
146
+ }
147
+ return out
app/context/microtopo.py ADDED
@@ -0,0 +1,274 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """LiDAR/DEM-derived micro-topography specialist.
2
+
3
+ Reads a window from a precomputed NYC-wide DEM (data/nyc_dem_30m.tif)
4
+ fetched from USGS 3DEP via py3dep. Computes per-address terrain numbers
5
+ that the static FEMA/DEP scenario maps don't expose.
6
+
7
+ Metrics (all derived from the same small AOI raster):
8
+
9
+ point_elev_m elevation at the address (m)
10
+ rel_elev_pct_750m percentile of point elev in a 750-m radius
11
+ rel_elev_pct_200m percentile of point elev in a 200-m radius
12
+ (block-scale "is this a bowl?")
13
+ basin_relief_m max-elev in 750-m AOI minus point elev
14
+ aoi_min_m, aoi_max_m for context
15
+ resolution_m
16
+
17
+ We deliberately stop at "shape-of-the-terrain" metrics rather than full
18
+ hydrology β€” depression-fill / D8 flow accumulation on a flat coastal
19
+ DEM are noisy and slow. Percentile + relief is what the reconciler
20
+ actually needs to write a useful sentence.
21
+ """
22
+ from __future__ import annotations
23
+
24
+ import logging
25
+ import warnings
26
+ from dataclasses import dataclass
27
+ from pathlib import Path
28
+
29
+ import numpy as np
30
+
31
+ warnings.filterwarnings("ignore")
32
+
33
+ log = logging.getLogger("riprap.microtopo")
34
+
35
+ DOC_ID = "microtopo"
36
+ CITATION = "USGS 3DEP 30 m DEM (precomputed citywide GeoTIFF, WGS84)"
37
+
38
+ DATA_DIR = Path(__file__).resolve().parent.parent.parent / "data"
39
+ DEM_PATH = DATA_DIR / "nyc_dem_30m.tif"
40
+ TWI_PATH = DATA_DIR / "twi.tif"
41
+ HAND_PATH = DATA_DIR / "hand.tif"
42
+
43
+
44
+ @dataclass
45
+ class Microtopo:
46
+ point_elev_m: float
47
+ rel_elev_pct_750m: float # 0..100
48
+ rel_elev_pct_200m: float # 0..100
49
+ basin_relief_m: float
50
+ aoi_min_m: float
51
+ aoi_max_m: float
52
+ aoi_radius_m: int
53
+ resolution_m: int
54
+ # Hydrology indices computed on the same DEM (whitebox-workflows)
55
+ twi: float | None = None # Topographic Wetness Index, ln(SCA / tan(slope))
56
+ hand_m: float | None = None # Height Above Nearest Drainage (m)
57
+
58
+
59
+ def _percentile_in_window(arr: np.ndarray, iy: int, ix: int, point_val: float,
60
+ window_radius_cells: int) -> float:
61
+ H, W = arr.shape
62
+ y0 = max(0, iy - window_radius_cells)
63
+ y1 = min(H, iy + window_radius_cells + 1)
64
+ x0 = max(0, ix - window_radius_cells)
65
+ x1 = min(W, ix + window_radius_cells + 1)
66
+ sub = arr[y0:y1, x0:x1]
67
+ finite = sub[np.isfinite(sub)]
68
+ if finite.size == 0:
69
+ return float("nan")
70
+ return float((finite < point_val).sum()) / finite.size * 100.0
71
+
72
+
73
+ _DEM_CACHE: dict = {}
74
+
75
+
76
+ def _read_full_raster(path: Path) -> tuple[np.ndarray | None, dict | None]:
77
+ import rasterio
78
+ if not path.exists():
79
+ return None, None
80
+ with rasterio.open(path) as ds:
81
+ arr = ds.read(1).astype("float32")
82
+ nodata = ds.nodata
83
+ meta = {"H": ds.height, "W": ds.width,
84
+ "transform": ds.transform, "crs": ds.crs, "nodata": nodata}
85
+ if nodata is not None:
86
+ arr = np.where(arr == nodata, np.nan, arr)
87
+ return arr, meta
88
+
89
+
90
+ def _load_dem():
91
+ """Read the precomputed NYC DEM + TWI + HAND rasters into memory.
92
+
93
+ All three are aligned (same grid, same transform). We hold them as
94
+ numpy arrays so per-query slicing is safe under threading.
95
+ """
96
+ if "arr" in _DEM_CACHE:
97
+ return _DEM_CACHE
98
+ arr, meta = _read_full_raster(DEM_PATH)
99
+ if arr is None:
100
+ log.warning("microtopo DEM not found at %s β€” run scripts/fetch_nyc_dem.py", DEM_PATH)
101
+ return None
102
+ twi, _ = _read_full_raster(TWI_PATH)
103
+ hand, _ = _read_full_raster(HAND_PATH)
104
+ _DEM_CACHE.update({
105
+ "arr": arr, "H": meta["H"], "W": meta["W"],
106
+ "transform": meta["transform"], "crs": meta["crs"],
107
+ "twi": twi, "hand": hand,
108
+ })
109
+ note = []
110
+ if twi is not None: note.append(f"TWI {TWI_PATH.name}")
111
+ if hand is not None: note.append(f"HAND {HAND_PATH.name}")
112
+ log.info("microtopo: loaded NYC DEM %s (%dx%d, %s); aux: %s",
113
+ DEM_PATH.name, meta["H"], meta["W"], meta["crs"],
114
+ ", ".join(note) if note else "(none β€” algorithmic only)")
115
+ return _DEM_CACHE
116
+
117
+
118
+ def warm():
119
+ _load_dem()
120
+
121
+
122
+ def _row_col(transform, lat: float, lon: float) -> tuple[int, int]:
123
+ """Inverse-affine: WGS84 (lon,lat) -> raster (row, col).
124
+ Mirrors rasterio.transform.rowcol but without holding a dataset handle.
125
+ """
126
+ # Diagonal affine (north-up raster): x = a*col + c, y = e*row + f.
127
+ a, c = transform.a, transform.c
128
+ e, f = transform.e, transform.f
129
+ col = int(round((lon - c) / a))
130
+ row = int(round((lat - f) / e))
131
+ return row, col
132
+
133
+
134
+ def microtopo_at(lat: float, lon: float, radius_m: int = 750) -> Microtopo | None:
135
+ state = _load_dem()
136
+ if state is None:
137
+ return None
138
+ arr_full = state["arr"]
139
+ transform = state["transform"]
140
+
141
+ try:
142
+ row, col = _row_col(transform, lat, lon)
143
+ except Exception as e:
144
+ log.warning("microtopo index failed: %s", e)
145
+ return None
146
+
147
+ res_m = abs(transform.a) * 111_000.0 * np.cos(np.radians(lat))
148
+ cells_radius = max(2, int(np.ceil(radius_m / max(res_m, 1.0))))
149
+
150
+ H, W = state["H"], state["W"]
151
+ y0 = max(0, row - cells_radius); y1 = min(H, row + cells_radius + 1)
152
+ x0 = max(0, col - cells_radius); x1 = min(W, col + cells_radius + 1)
153
+ if y1 <= y0 or x1 <= x0:
154
+ return None
155
+
156
+ arr = arr_full[y0:y1, x0:x1].copy()
157
+
158
+ iy = row - y0
159
+ ix = col - x0
160
+ if not (0 <= iy < arr.shape[0] and 0 <= ix < arr.shape[1]):
161
+ return None
162
+
163
+ point_elev = float(arr[iy, ix])
164
+ if not np.isfinite(point_elev):
165
+ for r in range(1, 6):
166
+ ya, yb = max(0, iy - r), min(arr.shape[0], iy + r + 1)
167
+ xa, xb = max(0, ix - r), min(arr.shape[1], ix + r + 1)
168
+ sub = arr[ya:yb, xa:xb]
169
+ if np.isfinite(sub).any():
170
+ point_elev = float(np.nanmean(sub))
171
+ break
172
+ else:
173
+ return None
174
+
175
+ finite = arr[np.isfinite(arr)]
176
+ if finite.size == 0:
177
+ return None
178
+ aoi_min = float(finite.min())
179
+ aoi_max = float(finite.max())
180
+
181
+ pct_750 = float((finite < point_elev).sum()) / finite.size * 100.0
182
+ cells_200m = max(1, int(round(200 / max(res_m, 1.0))))
183
+ pct_200 = _percentile_in_window(arr, iy, ix, point_elev, cells_200m)
184
+
185
+ twi_arr = state.get("twi")
186
+ hand_arr = state.get("hand")
187
+ twi_v: float | None = None
188
+ hand_v: float | None = None
189
+ if twi_arr is not None and 0 <= row < H and 0 <= col < W:
190
+ v = float(twi_arr[row, col])
191
+ twi_v = round(v, 2) if np.isfinite(v) else None
192
+ if hand_arr is not None and 0 <= row < H and 0 <= col < W:
193
+ v = float(hand_arr[row, col])
194
+ hand_v = round(v, 2) if np.isfinite(v) else None
195
+
196
+ return Microtopo(
197
+ point_elev_m=round(point_elev, 2),
198
+ rel_elev_pct_750m=round(pct_750, 1),
199
+ rel_elev_pct_200m=round(pct_200, 1),
200
+ basin_relief_m=round(aoi_max - point_elev, 2),
201
+ aoi_min_m=round(aoi_min, 2),
202
+ aoi_max_m=round(aoi_max, 2),
203
+ aoi_radius_m=radius_m,
204
+ resolution_m=int(round(res_m)),
205
+ twi=twi_v,
206
+ hand_m=hand_v,
207
+ )
208
+
209
+
210
+ def microtopo_for_polygon(polygon, polygon_crs: str = "EPSG:4326") -> dict | None:
211
+ """Polygon-mode aggregation: distributional summary of the DEM/HAND/TWI
212
+ rasters clipped to the polygon. Returns medians + fraction of cells
213
+ in flood-prone bands. Used for neighborhood-mode queries."""
214
+ state = _load_dem()
215
+ if state is None:
216
+ return None
217
+ try:
218
+ import rasterio
219
+ from rasterio.mask import mask as rio_mask
220
+ except Exception:
221
+ return None
222
+ import geopandas as gpd
223
+
224
+ poly = gpd.GeoDataFrame(geometry=[polygon], crs=polygon_crs).to_crs("EPSG:4326")
225
+ geom = [poly.iloc[0].geometry.__geo_interface__]
226
+
227
+ def _stats(path: Path) -> dict | None:
228
+ if not path.exists():
229
+ return None
230
+ try:
231
+ with rasterio.open(path) as src:
232
+ clipped, _ = rio_mask(src, geom, crop=True, filled=False)
233
+ arr = clipped[0]
234
+ vals = arr.compressed() if hasattr(arr, "compressed") else arr.flatten()
235
+ vals = vals[np.isfinite(vals)]
236
+ if vals.size == 0:
237
+ return None
238
+ return {
239
+ "n_cells": int(vals.size),
240
+ "min": float(np.min(vals)),
241
+ "median": float(np.median(vals)),
242
+ "p10": float(np.percentile(vals, 10)),
243
+ "p90": float(np.percentile(vals, 90)),
244
+ "max": float(np.max(vals)),
245
+ "raw": vals,
246
+ }
247
+ except Exception as e:
248
+ log.warning("polygon raster mask failed for %s: %r", path.name, e)
249
+ return None
250
+
251
+ elev = _stats(DEM_PATH)
252
+ hand = _stats(HAND_PATH)
253
+ twi = _stats(TWI_PATH)
254
+ if elev is None:
255
+ return None
256
+
257
+ # Fraction of polygon cells in canonical flood-prone bands
258
+ frac_hand_lt1 = (
259
+ round(float((hand["raw"] < 1.0).mean()), 4) if hand else None
260
+ )
261
+ frac_twi_gt10 = (
262
+ round(float((twi["raw"] > 10.0).mean()), 4) if twi else None
263
+ )
264
+ return {
265
+ "n_cells": elev["n_cells"],
266
+ "elev_min_m": round(elev["min"], 2),
267
+ "elev_median_m": round(elev["median"], 2),
268
+ "elev_p10_m": round(elev["p10"], 2),
269
+ "elev_max_m": round(elev["max"], 2),
270
+ "hand_median_m": round(hand["median"], 2) if hand else None,
271
+ "twi_median": round(twi["median"], 2) if twi else None,
272
+ "frac_hand_lt1": frac_hand_lt1,
273
+ "frac_twi_gt10": frac_twi_gt10,
274
+ }
app/context/noaa_tides.py ADDED
@@ -0,0 +1,110 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """NOAA CO-OPS Tides & Currents β€” live coastal water level.
2
+
3
+ api.tidesandcurrents.noaa.gov, no auth, 6-min cadence.
4
+
5
+ We pick the nearest of three NYC-region stations to the queried address:
6
+ - 8518750 The Battery, NY
7
+ - 8516945 Kings Point, NY (Long Island Sound entrance)
8
+ - 8531680 Sandy Hook, NJ (NY Harbor approach)
9
+
10
+ The verified-water-level API returns instantaneous water elevation
11
+ relative to MLLW (Mean Lower Low Water β€” the local tidal datum). To
12
+ distinguish "high tide" from "storm surge" we also fetch the published
13
+ predicted tide and report the residual.
14
+ """
15
+ from __future__ import annotations
16
+
17
+ from dataclasses import dataclass
18
+ from math import asin, cos, radians, sin, sqrt
19
+
20
+ import httpx
21
+
22
+ DOC_ID = "noaa_tides"
23
+ CITATION = "NOAA CO-OPS Tides & Currents (api.tidesandcurrents.noaa.gov)"
24
+ URL = "https://api.tidesandcurrents.noaa.gov/api/prod/datagetter"
25
+
26
+ STATIONS = [
27
+ # (id, name, lat, lon)
28
+ # NYC harbor + Long Island Sound
29
+ ("8518750", "The Battery, NY", 40.7006, -74.0142),
30
+ ("8516945", "Kings Point, NY", 40.8103, -73.7649),
31
+ ("8531680", "Sandy Hook, NJ", 40.4669, -74.0094),
32
+ # Hudson tidal corridor (head-of-tide is Troy / Albany; Hudson is tidal
33
+ # all the way up to the Federal Lock at Troy)
34
+ ("8518995", "Albany, NY (Hudson)", 42.6469, -73.7464),
35
+ ("8518962", "Turkey Point Hudson, NY", 41.7569, -73.9433),
36
+ ("8519483", "West Point, NY", 41.3845, -73.9536),
37
+ ]
38
+
39
+
40
+ @dataclass
41
+ class TideReading:
42
+ station_id: str
43
+ station_name: str
44
+ distance_km: float
45
+ observed_ft: float | None # current water level above MLLW
46
+ predicted_ft: float | None # astronomical prediction at same instant
47
+ residual_ft: float | None # observed - predicted (β‰ˆ storm surge)
48
+ obs_time: str | None
49
+ error: str | None = None
50
+
51
+
52
+ def _haversine_km(lat1, lon1, lat2, lon2) -> float:
53
+ R = 6371.0
54
+ p1, p2 = radians(lat1), radians(lat2)
55
+ dp = radians(lat2 - lat1); dl = radians(lon2 - lon1)
56
+ a = sin(dp/2)**2 + cos(p1)*cos(p2)*sin(dl/2)**2
57
+ return 2 * R * asin(sqrt(a))
58
+
59
+
60
+ def _nearest_station(lat: float, lon: float):
61
+ return min(STATIONS, key=lambda s: _haversine_km(lat, lon, s[2], s[3]))
62
+
63
+
64
+ def _fetch(station_id: str, product: str) -> dict:
65
+ r = httpx.get(URL, params={
66
+ "date": "latest", "station": station_id, "product": product,
67
+ "datum": "MLLW", "units": "english", "time_zone": "lst_ldt",
68
+ "format": "json",
69
+ }, timeout=8.0)
70
+ r.raise_for_status()
71
+ return r.json()
72
+
73
+
74
+ def reading_at(lat: float, lon: float) -> TideReading:
75
+ sid, name, slat, slon = _nearest_station(lat, lon)
76
+ dist_km = round(_haversine_km(lat, lon, slat, slon), 1)
77
+ out = TideReading(station_id=sid, station_name=name, distance_km=dist_km,
78
+ observed_ft=None, predicted_ft=None, residual_ft=None,
79
+ obs_time=None)
80
+ try:
81
+ obs = _fetch(sid, "water_level").get("data") or []
82
+ pred = _fetch(sid, "predictions").get("predictions") or []
83
+ if obs:
84
+ out.observed_ft = round(float(obs[0]["v"]), 2)
85
+ out.obs_time = obs[0].get("t")
86
+ if pred:
87
+ out.predicted_ft = round(float(pred[0]["v"]), 2)
88
+ if out.observed_ft is not None and out.predicted_ft is not None:
89
+ out.residual_ft = round(out.observed_ft - out.predicted_ft, 2)
90
+ except Exception as e:
91
+ out.error = str(e)
92
+ return out
93
+
94
+
95
+ def summary_for_point(lat: float, lon: float) -> dict:
96
+ r = reading_at(lat, lon)
97
+ # Look up station coords for the map marker.
98
+ sta = next((s for s in STATIONS if s[0] == r.station_id), None)
99
+ return {
100
+ "station_id": r.station_id,
101
+ "station_name": r.station_name,
102
+ "station_lat": sta[2] if sta else None,
103
+ "station_lon": sta[3] if sta else None,
104
+ "distance_km": r.distance_km,
105
+ "observed_ft_mllw": r.observed_ft,
106
+ "predicted_ft_mllw": r.predicted_ft,
107
+ "residual_ft": r.residual_ft,
108
+ "obs_time": r.obs_time,
109
+ "error": r.error,
110
+ }
app/context/npcc4_slr.py ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """NPCC4 sea-level rise projections for NYC (static lookup).
2
+
3
+ Source: New York City Panel on Climate Change 4th Assessment (2024),
4
+ Chapter 3, Table 3.2 β€” sea-level rise relative to 2000–2004 baseline,
5
+ Battery Tide Gauge (NOAA 8518750), primary NYC harbor reference.
6
+
7
+ Values are in inches above the 2000–2004 mean. The NPCC4 uses a
8
+ probabilistic framework across RCP/SSP scenarios; the table excerpted
9
+ here represents the "likely range" (10th–90th) plus the high-end
10
+ "extreme" scenario (99th).
11
+ """
12
+
13
+ DOC_ID = "npcc4_slr"
14
+ CITATION = (
15
+ "New York City Panel on Climate Change 4th Assessment (NPCC4 2024), "
16
+ "Chapter 3 β€” Sea Level Rise, Table 3.2. "
17
+ "Published by the New York Academy of Sciences. "
18
+ "Reference gauge: NOAA Battery (8518750), baseline 2000–2004."
19
+ )
20
+
21
+ # Sea-level rise projections in INCHES above the 2000–2004 baseline,
22
+ # Battery Tide Gauge. Percentiles: 10th (low), 50th (mid), 90th (high),
23
+ # 99th (extreme). All values from NPCC4 (2024) Ch. 3 Table 3.2.
24
+ _TABLE_IN = {
25
+ 2050: {10: 8, 50: 15, 90: 29, 99: 40},
26
+ 2100: {10: 13, 50: 31, 90: 65, 99: 96},
27
+ }
28
+
29
+
30
+ def _in_to_m(inches: float) -> float:
31
+ return round(inches * 0.0254, 2)
32
+
33
+
34
+ def get_projections() -> dict:
35
+ """Return NPCC4 SLR projection dict, always available (static table)."""
36
+ result: dict = {"available": True, "baseline": "2000–2004", "gauge": "NOAA Battery (8518750)"}
37
+ for year, pcts in _TABLE_IN.items():
38
+ result[str(year)] = {
39
+ str(pct): {"in": v, "m": _in_to_m(v)}
40
+ for pct, v in pcts.items()
41
+ }
42
+ return result
app/context/nws_alerts.py ADDED
@@ -0,0 +1,71 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """NWS API β€” active alerts at a point.
2
+
3
+ api.weather.gov/alerts/active?point={lat},{lon}, no auth, JSON.
4
+ A User-Agent header is required (NWS rate-limits anonymous traffic).
5
+
6
+ We surface only flood-relevant categories so the doc the reconciler
7
+ sees is short and on-topic.
8
+ """
9
+ from __future__ import annotations
10
+
11
+ from typing import Any
12
+
13
+ import httpx
14
+
15
+ DOC_ID = "nws_alerts"
16
+ CITATION = "NWS public alert API (api.weather.gov/alerts)"
17
+
18
+ USER_AGENT = "Riprap-NYC/0.1 (civic-flood-tool; +https://huggingface.co/spaces/msradam/riprap-nyc)"
19
+
20
+ _FLOOD_EVENT_KEYWORDS = (
21
+ "flood", "flash flood", "coastal flood", "high surf", "storm surge",
22
+ "hurricane", "tropical storm", "tornado warning", # high-impact context
23
+ "rip current",
24
+ )
25
+
26
+
27
+ def _is_flood_relevant(event_name: str) -> bool:
28
+ e = (event_name or "").lower()
29
+ return any(k in e for k in _FLOOD_EVENT_KEYWORDS)
30
+
31
+
32
+ def alerts_at(lat: float, lon: float) -> list[dict[str, Any]]:
33
+ r = httpx.get(
34
+ "https://api.weather.gov/alerts/active",
35
+ params={"point": f"{lat:.4f},{lon:.4f}"},
36
+ headers={"User-Agent": USER_AGENT, "Accept": "application/geo+json"},
37
+ timeout=8.0,
38
+ )
39
+ r.raise_for_status()
40
+ out = []
41
+ for f in r.json().get("features", []):
42
+ p = f.get("properties", {}) or {}
43
+ event = p.get("event") or ""
44
+ if not _is_flood_relevant(event):
45
+ continue
46
+ out.append({
47
+ "id": p.get("id"),
48
+ "event": event,
49
+ "severity": p.get("severity"),
50
+ "urgency": p.get("urgency"),
51
+ "certainty": p.get("certainty"),
52
+ "headline": p.get("headline"),
53
+ "sent": p.get("sent"),
54
+ "effective": p.get("effective"),
55
+ "expires": p.get("expires"),
56
+ "sender_name": p.get("senderName"),
57
+ "areaDesc": p.get("areaDesc"),
58
+ })
59
+ return out
60
+
61
+
62
+ def summary_for_point(lat: float, lon: float) -> dict:
63
+ try:
64
+ active = alerts_at(lat, lon)
65
+ except Exception as e:
66
+ return {"n_active": 0, "alerts": [], "error": str(e)}
67
+ return {
68
+ "n_active": len(active),
69
+ "alerts": active,
70
+ "error": None,
71
+ }
app/context/nws_obs.py ADDED
@@ -0,0 +1,108 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """NWS station observations β€” latest hourly METAR for the nearest NYC airport.
2
+
3
+ api.weather.gov/stations/{id}/observations/latest.
4
+
5
+ Five NYC-region ASOS stations cover the city; we pick the nearest.
6
+ Most useful field for flood context is hourly precipitation (the
7
+ `precipitationLastHour` quantity, mm). The latest observation is
8
+ typically <60 min old.
9
+ """
10
+ from __future__ import annotations
11
+
12
+ from dataclasses import dataclass
13
+ from math import asin, cos, radians, sin, sqrt
14
+
15
+ import httpx
16
+
17
+ DOC_ID = "nws_obs"
18
+ CITATION = "NWS station observations API (api.weather.gov/stations)"
19
+
20
+ USER_AGENT = "Riprap-NYC/0.1 (civic-flood-tool; +https://huggingface.co/spaces/msradam/riprap-nyc)"
21
+
22
+ # NYC + Hudson Corridor ASOS stations. Picker is haversine-nearest, so adding
23
+ # upstate stations enables Albany / Poughkeepsie / Newburgh queries without
24
+ # breaking NYC behaviour (NYC stations stay closer for NYC lat/lon).
25
+ STATIONS = [
26
+ # NYC region
27
+ ("KNYC", "Central Park, NY", 40.7794, -73.9692),
28
+ ("KLGA", "LaGuardia Airport, NY", 40.7794, -73.8800),
29
+ ("KJFK", "JFK Airport, NY", 40.6413, -73.7781),
30
+ ("KEWR", "Newark Liberty, NJ", 40.6925, -74.1687),
31
+ ("KFRG", "Republic Farmingdale, NY", 40.7288, -73.4134),
32
+ # Hudson Corridor (south β†’ north)
33
+ ("KHPN", "White Plains, NY", 41.0670, -73.7076),
34
+ ("KSWF", "Newburgh-Stewart, NY", 41.5042, -74.1048),
35
+ ("KPOU", "Poughkeepsie, NY", 41.6262, -73.8842),
36
+ ("KALB", "Albany Intl, NY", 42.7475, -73.8025),
37
+ ]
38
+
39
+
40
+ @dataclass
41
+ class Obs:
42
+ station_id: str
43
+ station_name: str
44
+ distance_km: float
45
+ obs_time: str | None
46
+ temp_c: float | None
47
+ precip_last_hour_mm: float | None
48
+ precip_last_3h_mm: float | None
49
+ precip_last_6h_mm: float | None
50
+ error: str | None = None
51
+
52
+
53
+ def _haversine_km(lat1, lon1, lat2, lon2) -> float:
54
+ R = 6371.0
55
+ p1, p2 = radians(lat1), radians(lat2)
56
+ dp = radians(lat2 - lat1); dl = radians(lon2 - lon1)
57
+ a = sin(dp/2)**2 + cos(p1)*cos(p2)*sin(dl/2)**2
58
+ return 2 * R * asin(sqrt(a))
59
+
60
+
61
+ def _val_mm(props, key) -> float | None:
62
+ """NWS returns {value: ..., unitCode: 'wmoUnit:mm'} per quantity. Convert
63
+ to mm; if value is null, return None."""
64
+ q = (props or {}).get(key) or {}
65
+ v = q.get("value")
66
+ if v is None:
67
+ return None
68
+ return round(float(v), 2)
69
+
70
+
71
+ def obs_at(lat: float, lon: float) -> Obs:
72
+ sid, name, slat, slon = min(STATIONS, key=lambda s: _haversine_km(lat, lon, s[2], s[3]))
73
+ dist_km = round(_haversine_km(lat, lon, slat, slon), 1)
74
+ out = Obs(station_id=sid, station_name=name, distance_km=dist_km,
75
+ obs_time=None, temp_c=None,
76
+ precip_last_hour_mm=None, precip_last_3h_mm=None,
77
+ precip_last_6h_mm=None)
78
+ try:
79
+ r = httpx.get(
80
+ f"https://api.weather.gov/stations/{sid}/observations/latest",
81
+ headers={"User-Agent": USER_AGENT, "Accept": "application/geo+json"},
82
+ timeout=8.0,
83
+ )
84
+ r.raise_for_status()
85
+ p = r.json().get("properties", {}) or {}
86
+ out.obs_time = p.get("timestamp")
87
+ out.temp_c = _val_mm(p, "temperature")
88
+ out.precip_last_hour_mm = _val_mm(p, "precipitationLastHour")
89
+ out.precip_last_3h_mm = _val_mm(p, "precipitationLast3Hours")
90
+ out.precip_last_6h_mm = _val_mm(p, "precipitationLast6Hours")
91
+ except Exception as e:
92
+ out.error = str(e)
93
+ return out
94
+
95
+
96
+ def summary_for_point(lat: float, lon: float) -> dict:
97
+ o = obs_at(lat, lon)
98
+ return {
99
+ "station_id": o.station_id,
100
+ "station_name": o.station_name,
101
+ "distance_km": o.distance_km,
102
+ "obs_time": o.obs_time,
103
+ "temp_c": o.temp_c,
104
+ "precip_last_hour_mm": o.precip_last_hour_mm,
105
+ "precip_last_3h_mm": o.precip_last_3h_mm,
106
+ "precip_last_6h_mm": o.precip_last_6h_mm,
107
+ "error": o.error,
108
+ }
app/context/nyc311.py ADDED
@@ -0,0 +1,161 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """NYC 311 β€” flood-related complaints around a point.
2
+
3
+ Live dataset: erm2-nwe9. Filter by descriptor (the flood signal is in
4
+ descriptor, not complaint_type) within a buffer.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ from collections import Counter
9
+ from dataclasses import dataclass
10
+ from datetime import datetime, timedelta, timezone
11
+
12
+ import httpx
13
+
14
+ URL = "https://data.cityofnewyork.us/resource/erm2-nwe9.json"
15
+ DOC_ID = "nyc311"
16
+ CITATION = "NYC 311 service requests (Socrata erm2-nwe9, 2010-present)"
17
+
18
+ FLOOD_DESCRIPTORS = [
19
+ "Street Flooding (SJ)",
20
+ "Sewer Backup (Use Comments) (SA)",
21
+ "Catch Basin Clogged/Flooding (Use Comments) (SC)",
22
+ "Highway Flooding (SH)",
23
+ "Manhole Overflow (Use Comments) (SA1)",
24
+ "Flooding on Street",
25
+ "RAIN GARDEN FLOODING (SRGFLD)",
26
+ ]
27
+
28
+ _DESC_CLAUSE = "(" + " OR ".join(f"descriptor='{d}'" for d in FLOOD_DESCRIPTORS) + ")"
29
+
30
+
31
+ @dataclass
32
+ class Complaint:
33
+ unique_key: str
34
+ descriptor: str
35
+ created_date: str
36
+ address: str | None
37
+ status: str | None
38
+ lat: float | None = None
39
+ lon: float | None = None
40
+
41
+
42
+ def complaints_near(lat: float, lon: float, radius_m: float = 200,
43
+ since: datetime | None = None,
44
+ limit: int = 1000) -> list[Complaint]:
45
+ where = f"{_DESC_CLAUSE} AND within_circle(location, {lat}, {lon}, {radius_m})"
46
+ if since:
47
+ # Socrata floating-timestamp: drop tz suffix
48
+ ts = since.replace(tzinfo=None).isoformat(timespec="seconds")
49
+ where += f" AND created_date >= '{ts}'"
50
+ r = httpx.get(URL, params={
51
+ "$select": "unique_key, descriptor, created_date, incident_address, "
52
+ "status, latitude, longitude",
53
+ "$where": where,
54
+ "$order": "created_date desc",
55
+ "$limit": str(limit),
56
+ }, timeout=30)
57
+ r.raise_for_status()
58
+ out = []
59
+ for row in r.json():
60
+ lat = row.get("latitude")
61
+ lon = row.get("longitude")
62
+ try:
63
+ lat = float(lat) if lat is not None else None
64
+ lon = float(lon) if lon is not None else None
65
+ except Exception:
66
+ lat, lon = None, None
67
+ out.append(Complaint(
68
+ unique_key=row.get("unique_key", ""),
69
+ descriptor=row.get("descriptor", ""),
70
+ created_date=row.get("created_date", ""),
71
+ address=row.get("incident_address"),
72
+ status=row.get("status"),
73
+ lat=lat, lon=lon,
74
+ ))
75
+ return out
76
+
77
+
78
+ def summary_for_point(lat: float, lon: float, radius_m: float = 200,
79
+ years: int = 5) -> dict:
80
+ since = datetime.now(timezone.utc) - timedelta(days=365 * years)
81
+ cs = complaints_near(lat, lon, radius_m, since=since, limit=2000)
82
+ return _summarize(cs, years=years, radius_m=radius_m)
83
+
84
+
85
+ def complaints_in_polygon(polygon, polygon_crs: str = "EPSG:4326",
86
+ since: datetime | None = None,
87
+ limit: int = 5000,
88
+ simplify_tolerance: float = 0.0005) -> list[Complaint]:
89
+ """Pull flood-related complaints inside an arbitrary polygon via
90
+ Socrata's `within_polygon(location, 'MULTIPOLYGON(...)')` predicate.
91
+
92
+ NYC NTA polygons can have thousands of vertices and exceed Socrata's
93
+ URL length limit (414). We simplify in EPSG:4326 with a default
94
+ ~50 m tolerance, which collapses vertex count ~10-20Γ— without
95
+ materially changing the contained-points result.
96
+
97
+ Polygon must be EPSG:4326 (lat/lon) for the Socrata query.
98
+ """
99
+ import geopandas as gpd
100
+ g = gpd.GeoDataFrame(geometry=[polygon], crs=polygon_crs).to_crs("EPSG:4326")
101
+ geom = g.iloc[0].geometry.simplify(simplify_tolerance, preserve_topology=True)
102
+ wkt = geom.wkt
103
+ where = f"{_DESC_CLAUSE} AND within_polygon(location, '{wkt}')"
104
+ if since:
105
+ ts = since.replace(tzinfo=None).isoformat(timespec="seconds")
106
+ where += f" AND created_date >= '{ts}'"
107
+ r = httpx.get(URL, params={
108
+ "$select": "unique_key, descriptor, created_date, incident_address, status",
109
+ "$where": where,
110
+ "$order": "created_date desc",
111
+ "$limit": str(limit),
112
+ }, timeout=60)
113
+ r.raise_for_status()
114
+ return [
115
+ Complaint(
116
+ unique_key=row.get("unique_key", ""),
117
+ descriptor=row.get("descriptor", ""),
118
+ created_date=row.get("created_date", ""),
119
+ address=row.get("incident_address"),
120
+ status=row.get("status"),
121
+ )
122
+ for row in r.json()
123
+ ]
124
+
125
+
126
+ def summary_for_polygon(polygon, polygon_crs: str = "EPSG:4326",
127
+ years: int = 5) -> dict:
128
+ """Polygon-mode aggregation: counts of flood-related 311 complaints
129
+ inside the polygon over the trailing window."""
130
+ since = datetime.now(timezone.utc) - timedelta(days=365 * years)
131
+ cs = complaints_in_polygon(polygon, polygon_crs=polygon_crs, since=since)
132
+ return _summarize(cs, years=years, radius_m=None)
133
+
134
+
135
+ def _summarize(cs: list[Complaint], years: int, radius_m: float | None) -> dict:
136
+ by_year: Counter = Counter(c.created_date[:4] for c in cs if c.created_date)
137
+ by_descriptor: Counter = Counter(c.descriptor for c in cs)
138
+ # Cap at 60 most-recent points for the map layer β€” keeps the SSE
139
+ # payload small while still showing meaningful clustering.
140
+ points = [
141
+ {"lat": c.lat, "lon": c.lon,
142
+ "descriptor": c.descriptor,
143
+ "date": c.created_date[:10],
144
+ "address": c.address}
145
+ for c in cs[:60]
146
+ if c.lat is not None and c.lon is not None
147
+ ]
148
+ return {
149
+ "n": len(cs),
150
+ "radius_m": radius_m,
151
+ "years": years,
152
+ "by_year": dict(sorted(by_year.items())),
153
+ "by_descriptor": dict(by_descriptor.most_common(6)),
154
+ "most_recent": [
155
+ {"date": c.created_date[:10],
156
+ "descriptor": c.descriptor,
157
+ "address": c.address}
158
+ for c in cs[:5]
159
+ ],
160
+ "points": points,
161
+ }
app/context/terramind_nyc.py ADDED
@@ -0,0 +1,485 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """TerraMind-NYC adapters β€” LULC and Buildings inference for NYC chips.
2
+
3
+ Wraps the Apache-2.0 [`msradam/TerraMind-NYC-Adapters`](https://huggingface.co/msradam/TerraMind-NYC-Adapters)
4
+ LoRA family fine-tuned on NYC EO chips (Sentinel-2 L2A + Sentinel-1 RTC
5
+ + Copernicus DEM, temporal stack of 4) on AMD MI300X via AMD Developer
6
+ Cloud. Exposes two specialist entry points:
7
+
8
+ lulc(s2l2a, s1rtc, dem) -> 5-class macro NYC LULC mask
9
+ buildings(s2l2a, s1rtc, dem) -> binary NYC building footprint mask
10
+
11
+ The base TerraMind 1.0 weights are downloaded by terratorch on first
12
+ call; the LoRA adapter + UNet decoder weights come from the HF repo and
13
+ are cached to `~/.cache/huggingface/hub`.
14
+
15
+ CHIP-SIZE TRAP. TerraMind's positional embeddings don't generalise off
16
+ its training resolution (224Γ—224). Calling `task.model({...})` on a
17
+ chip β‰  224Γ—224 produces silent garbage. We therefore wrap inference
18
+ with `terratorch.tasks.tiled_inference.tiled_inference`, which slides
19
+ a 224Γ—224 crop window across the chip and stitches per-window logits.
20
+ This matches the patch in
21
+ `experiments/18_terramind_nyc_lora/shared/inference_ensemble.py` that
22
+ the plan flags as required for production.
23
+
24
+ Gated by RIPRAP_TERRAMIND_NYC_ENABLE β€” deployments without the deps
25
+ installed (HF Spaces' Py3.10 cone, plain Ollama dev VMs) silently no-op
26
+ through the same skipped-result shape every other heavy specialist
27
+ emits.
28
+
29
+ This module does NOT fetch its own S2/S1/DEM chips. C4 wires it into
30
+ the FSM with a shared chip cache so the LULC and Buildings calls
31
+ don't each refetch ~150 MB of imagery.
32
+ """
33
+ from __future__ import annotations
34
+
35
+ import logging
36
+ import os
37
+ import threading
38
+ import time
39
+ from typing import Any
40
+
41
+ log = logging.getLogger("riprap.terramind_nyc")
42
+
43
+ ENABLE = os.environ.get("RIPRAP_TERRAMIND_NYC_ENABLE", "1").lower() in ("1", "true", "yes")
44
+ DEVICE = os.environ.get("RIPRAP_TERRAMIND_NYC_DEVICE", "cpu")
45
+ ADAPTERS_REPO = "msradam/TerraMind-NYC-Adapters"
46
+
47
+ # Per-task config knobs the HF README's quick-start fixes for these
48
+ # adapters. Mirrored from experiments/18_terramind_nyc_lora/adapters/*/
49
+ # config.yaml so a single source of truth lives next to the inference
50
+ # code rather than being scraped from YAML at runtime.
51
+ ADAPTER_SPECS: dict[str, dict[str, Any]] = {
52
+ "lulc": {
53
+ "subdir": "lulc_nyc",
54
+ "num_classes": 5,
55
+ "class_labels": [
56
+ "Trees / vegetation",
57
+ "Cropland",
58
+ "Built / impervious",
59
+ "Bare ground",
60
+ "Water",
61
+ ],
62
+ },
63
+ "buildings": {
64
+ "subdir": "buildings_nyc",
65
+ "num_classes": 2,
66
+ # The decoder emits class 0 = background, class 1 = building.
67
+ "class_labels": ["Background", "Building footprint"],
68
+ },
69
+ }
70
+
71
+ # Tile-window size β€” TerraMind's training resolution. Stride < window
72
+ # yields overlap (smooths seams from window-boundary classification
73
+ # noise); 96 px overlap matches the experiments/18 ensemble.
74
+ TILE_SIZE = 224
75
+ TILE_STRIDE = 128
76
+
77
+ # One-shot lazy-init guards. The base TerraMind weights are heavy
78
+ # (~1.6 GB) and we want to load them once across LULC and Buildings.
79
+ _INIT_LOCK = threading.Lock()
80
+ _BASE_LOADED = False
81
+ _ADAPTERS: dict[str, Any] = {} # name -> built terratorch task on DEVICE
82
+
83
+
84
+ def _has_required_deps() -> tuple[bool, str | None]:
85
+ """Probe the heavy-EO deps. Same shape as prithvi_live's check β€”
86
+ a missing dep (terratorch / peft / safetensors / hf_hub) returns a
87
+ clean `skipped: deps_unavailable` outcome instead of a noisy
88
+ ModuleNotFoundError in the trace.
89
+
90
+ On the HF Space, terratorch's import chain itself can raise
91
+ RuntimeError("operator torchvision::nms does not exist") when the
92
+ torchvision binary extension can't load against our CPU torch
93
+ wheel. Treat that as 'unavailable' too β€” the local inference path
94
+ is dead-on-arrival there."""
95
+ missing: list[str] = []
96
+ for name in ("terratorch", "peft", "safetensors", "huggingface_hub",
97
+ "torch", "yaml"):
98
+ try:
99
+ __import__(name)
100
+ except ImportError:
101
+ missing.append(name)
102
+ except Exception as e:
103
+ # torchvision::nms RuntimeError, libcuda load failure, etc.
104
+ log.warning("terramind_nyc: %s import raised %s; treating as "
105
+ "unavailable", name, type(e).__name__)
106
+ missing.append(f"{name} ({type(e).__name__})")
107
+ if missing:
108
+ return False, ", ".join(missing)
109
+ return True, None
110
+
111
+
112
+ _DEPS_OK, _DEPS_MISSING = _has_required_deps()
113
+
114
+
115
+ def _ensure_adapter(adapter_name: str):
116
+ """Build the terratorch SemanticSegmentationTask, inject the LoRA
117
+ scaffold, load the published Ξ” + decoder weights, return the task.
118
+
119
+ Per-task tasks share the TerraMind base inside terratorch's model
120
+ factory β€” calling SemanticSegmentationTask twice loads the base
121
+ twice in fp32 (~3.3 GB resident on CPU). For a two-task family this
122
+ is acceptable; we don't need the cross-task weight sharing the
123
+ experiments/18 ensemble does. If memory becomes a problem, swap
124
+ this for a single-task / hot-swap-adapter implementation.
125
+ """
126
+ if adapter_name not in ADAPTER_SPECS:
127
+ raise KeyError(f"unknown adapter {adapter_name!r}; "
128
+ f"expected one of {list(ADAPTER_SPECS)}")
129
+ if adapter_name in _ADAPTERS:
130
+ return _ADAPTERS[adapter_name]
131
+
132
+ with _INIT_LOCK:
133
+ if adapter_name in _ADAPTERS:
134
+ return _ADAPTERS[adapter_name]
135
+
136
+ spec = ADAPTER_SPECS[adapter_name]
137
+ log.info("terramind_nyc: building task for %s", adapter_name)
138
+
139
+ from huggingface_hub import snapshot_download
140
+ from peft import LoraConfig, inject_adapter_in_model
141
+ from safetensors.torch import load_file
142
+ from terratorch.tasks import SemanticSegmentationTask
143
+
144
+ # 1. Pull the requested adapter subtree from the HF repo.
145
+ adapter_root = snapshot_download(
146
+ ADAPTERS_REPO,
147
+ allow_patterns=[f"{spec['subdir']}/*"],
148
+ )
149
+
150
+ # 2. Build the standard terratorch task with the same model_args
151
+ # the published HF_README quick-start uses.
152
+ task = SemanticSegmentationTask(
153
+ model_factory="EncoderDecoderFactory",
154
+ model_args=dict(
155
+ backbone="terramind_v1_base",
156
+ backbone_pretrained=True,
157
+ backbone_modalities=["S2L2A", "S1RTC", "DEM"],
158
+ backbone_use_temporal=True,
159
+ backbone_temporal_pooling="concat",
160
+ backbone_temporal_n_timestamps=4,
161
+ necks=[
162
+ {"name": "SelectIndices", "indices": [2, 5, 8, 11]},
163
+ {"name": "ReshapeTokensToImage", "remove_cls_token": False},
164
+ {"name": "LearnedInterpolateToPyramidal"},
165
+ ],
166
+ decoder="UNetDecoder",
167
+ decoder_channels=[512, 256, 128, 64],
168
+ head_dropout=0.1,
169
+ num_classes=spec["num_classes"],
170
+ ),
171
+ loss="ce", lr=1e-4, freeze_backbone=False, freeze_decoder=False,
172
+ )
173
+
174
+ # 3. Inject the LoRA scaffold the adapter weights were trained
175
+ # against. Same hyperparameters every adapter in this family
176
+ # used (see experiments/18 adapters/_template/config.yaml).
177
+ inject_adapter_in_model(LoraConfig(
178
+ r=16, lora_alpha=32, lora_dropout=0.05,
179
+ target_modules=["attn.qkv", "attn.proj"], bias="none",
180
+ ), task.model.encoder)
181
+
182
+ # 4. Restore Ξ” matrices (encoder LoRA) and the decoder/neck/head
183
+ # weights from the safetensors bundle. The encoder.* prefix
184
+ # is stripped because the encoder state-dict is rooted at
185
+ # the encoder module, not the task.
186
+ adapter_dir = f"{adapter_root}/{spec['subdir']}"
187
+ lora_state = load_file(f"{adapter_dir}/adapter_model.safetensors")
188
+ head_state = load_file(f"{adapter_dir}/decoder_head.safetensors")
189
+ encoder_state = {
190
+ k.removeprefix("encoder."): v
191
+ for k, v in lora_state.items() if k.startswith("encoder.")
192
+ }
193
+ task.model.encoder.load_state_dict(encoder_state, strict=False)
194
+ for sub in ("decoder", "neck", "head", "aux_heads"):
195
+ sub_state = {
196
+ k[len(sub) + 1:]: v
197
+ for k, v in head_state.items() if k.startswith(sub + ".")
198
+ }
199
+ if sub_state and hasattr(task.model, sub):
200
+ getattr(task.model, sub).load_state_dict(sub_state,
201
+ strict=False)
202
+
203
+ # 5. Move to the configured device. CUDA only if the caller
204
+ # asked AND a CUDA device is actually available β€” silently
205
+ # fall back to CPU otherwise.
206
+ target_device = DEVICE
207
+ if target_device == "cuda":
208
+ import torch
209
+ if not torch.cuda.is_available():
210
+ log.warning("terramind_nyc: CUDA unavailable, falling back to CPU")
211
+ target_device = "cpu"
212
+ task = task.to(target_device).eval()
213
+
214
+ _ADAPTERS[adapter_name] = task
215
+ log.info("terramind_nyc: %s ready on %s", adapter_name, target_device)
216
+ return task
217
+
218
+
219
+ def _tiled_predict(task, modality_chips: dict, num_classes: int):
220
+ """Run the task's encoder-decoder forward in 224Γ—224 tiles, returning
221
+ a (1, num_classes, H, W) logits tensor stitched from the windows.
222
+
223
+ TerraMind's positional embeddings are tied to the 224Γ—224 training
224
+ resolution. terratorch's tiled_inference helper slides a window
225
+ across the input modalities (it accepts a dict of per-modality
226
+ tensors as long as all modalities share HΓ—W), runs the model on
227
+ each crop, and averages overlapping logits. Without it, larger
228
+ chips return silent garbage; smaller chips error on the encoder
229
+ ViT.
230
+ """
231
+ import torch
232
+ from terratorch.tasks.tiled_inference import tiled_inference
233
+
234
+ # tiled_inference invokes `model_forward(patch)` per tile. The task
235
+ # model returns a ModelOutput-like with .output OR a plain tensor;
236
+ # coerce to tensor either way.
237
+ def _forward(x, **_extra):
238
+ out = task.model(x)
239
+ return out.output if hasattr(out, "output") else out
240
+
241
+ with torch.no_grad():
242
+ logits = tiled_inference(
243
+ _forward,
244
+ modality_chips,
245
+ out_channels=num_classes,
246
+ h_crop=TILE_SIZE,
247
+ w_crop=TILE_SIZE,
248
+ h_stride=TILE_STRIDE,
249
+ w_stride=TILE_STRIDE,
250
+ average_patches=True,
251
+ blend_overlaps=True,
252
+ padding="reflect",
253
+ )
254
+ return logits
255
+
256
+
257
+ def _summarize_lulc(pred, class_labels: list[str]) -> dict[str, Any]:
258
+ """Per-class pixel fraction + dominant class from an integer mask."""
259
+ import numpy as np
260
+ pred_np = pred.detach().cpu().numpy() if hasattr(pred, "detach") else np.asarray(pred)
261
+ flat = pred_np.reshape(-1)
262
+ n = max(int(flat.size), 1)
263
+ fractions: dict[str, float] = {}
264
+ for idx, label in enumerate(class_labels):
265
+ pct = 100.0 * float((flat == idx).sum()) / n
266
+ if pct > 0:
267
+ fractions[label] = round(pct, 2)
268
+ dominant_idx = int(max(range(len(class_labels)),
269
+ key=lambda i: int((flat == i).sum())))
270
+ return {
271
+ "ok": True,
272
+ "n_pixels": int(flat.size),
273
+ "shape": list(pred_np.shape),
274
+ "class_fractions": fractions,
275
+ "dominant_class": class_labels[dominant_idx],
276
+ "dominant_pct": fractions.get(class_labels[dominant_idx], 0.0),
277
+ }
278
+
279
+
280
+ def _summarize_buildings(pred, class_labels: list[str]) -> dict[str, Any]:
281
+ """Building-pixel coverage + simple connected-component count."""
282
+ import numpy as np
283
+ pred_np = pred.detach().cpu().numpy() if hasattr(pred, "detach") else np.asarray(pred)
284
+ mask = (pred_np == 1).astype("uint8")
285
+ n_total = max(int(mask.size), 1)
286
+ pct_built = 100.0 * float(mask.sum()) / n_total
287
+ # Connected-component count is a cheap signal of "how many distinct
288
+ # buildings does this chip cover" β€” useful for the briefing without
289
+ # paying for full polygonisation.
290
+ n_components: int | None = None
291
+ try:
292
+ from scipy.ndimage import label
293
+ _, n_components = label(mask)
294
+ except Exception: # scipy is optional in some HF Spaces build cones
295
+ log.debug("terramind_nyc: scipy.ndimage unavailable; "
296
+ "skipping component count")
297
+ return {
298
+ "ok": True,
299
+ "n_pixels": int(mask.size),
300
+ "shape": list(mask.shape),
301
+ "pct_buildings": round(pct_built, 2),
302
+ "n_building_components": n_components,
303
+ "class_labels": class_labels,
304
+ }
305
+
306
+
307
+ def _try_remote(adapter_name: str, modality_chips: dict) -> dict | None:
308
+ """POST to the riprap-models inference service if configured.
309
+
310
+ Returns:
311
+ - successful result dict on a 200/ok=True remote response
312
+ - {"ok": False, "skipped": "<reason>"} when remote was attempted
313
+ but failed (RemoteUnreachable, ok=False, or other error). The
314
+ caller MUST NOT fall through to local terratorch in this case
315
+ β€” local has been broken on the CPU-tier UI Spaces since the
316
+ torchvision binary mismatch landed, and we'd rather show a
317
+ clean "remote unreachable" reason than a noisy crash.
318
+ - None ONLY when remote isn't configured at all (caller may
319
+ legitimately try local then)."""
320
+ try:
321
+ from app import inference as _inf
322
+ if not _inf.remote_enabled():
323
+ return None
324
+ s2 = modality_chips.get("S2L2A")
325
+ s1 = modality_chips.get("S1RTC")
326
+ dem = modality_chips.get("DEM")
327
+ # The router serializes torch tensors to base64 numpy float32 β€”
328
+ # the chip cache hands us [B, C, T, H, W]; keep that shape, the
329
+ # service rebuilds the temporal stack on its end.
330
+ result = _inf.terramind(adapter_name, s2, s1, dem)
331
+ if not result.get("ok"):
332
+ err = result.get("error") or result.get("err") or "unknown"
333
+ return {"ok": False,
334
+ "skipped": f"remote terramind/{adapter_name} non-ok: {err}"}
335
+ result.setdefault("adapter", adapter_name)
336
+ result.setdefault("repo", ADAPTERS_REPO)
337
+ result["compute"] = f"remote Β· {result.get('device', 'gpu')}"
338
+ # Polygonize the prediction raster onto the chip's bounds so
339
+ # the map can paint the LULC / buildings overlay. Bounds come
340
+ # via the modality_chips dict β€” the eo_chip layer threads them
341
+ # through. Best-effort; never raises into the FSM.
342
+ bounds = modality_chips.get("bounds_4326") if modality_chips else None
343
+ pred_b64 = result.get("pred_b64")
344
+ pred_shape = result.get("pred_shape")
345
+ class_labels = result.get("class_labels")
346
+ if bounds and pred_b64 and pred_shape:
347
+ try:
348
+ from app.context._polygonize import (
349
+ polygonize_binary_mask, polygonize_class_raster,
350
+ )
351
+ if adapter_name == "buildings":
352
+ polys = polygonize_binary_mask(
353
+ pred_b64, pred_shape, tuple(bounds),
354
+ label="building", fill_color="#D62728",
355
+ simplify_tolerance=2e-5,
356
+ )
357
+ else:
358
+ polys = polygonize_class_raster(
359
+ pred_b64, pred_shape, class_labels, tuple(bounds),
360
+ simplify_tolerance=2e-5,
361
+ )
362
+ result["polygons_geojson"] = polys
363
+ except Exception:
364
+ log.exception("terramind/%s: polygonize failed", adapter_name)
365
+ result["polygons_geojson"] = None
366
+ return result
367
+ except _inf.RemoteUnreachable as e:
368
+ log.info("terramind/%s: remote unreachable (%s)", adapter_name, e)
369
+ return {"ok": False,
370
+ "skipped": f"remote terramind/{adapter_name} unreachable: {e}"}
371
+ except Exception as e:
372
+ log.exception("terramind/%s: remote call failed", adapter_name)
373
+ return {"ok": False,
374
+ "skipped": f"remote terramind/{adapter_name} error: "
375
+ f"{type(e).__name__}: {e}"}
376
+
377
+
378
+ def _run(adapter_name: str, modality_chips: dict, summarizer):
379
+ """Common boilerplate: gate, time, [remote attempt], load, tiled
380
+ predict, summarize."""
381
+ if not ENABLE:
382
+ return {"ok": False,
383
+ "skipped": "RIPRAP_TERRAMIND_NYC_ENABLE=0"}
384
+
385
+ # v0.4.5 β€” try remote first. The remote service has its own deps,
386
+ # so this path works even when local _DEPS_OK is False (the most
387
+ # common HF Spaces case until terratorch + peft are baked in).
388
+ remote = _try_remote(adapter_name, modality_chips or {})
389
+ if remote is not None:
390
+ return remote
391
+
392
+ if not _DEPS_OK:
393
+ return {"ok": False,
394
+ "skipped": f"deps unavailable on this deployment: "
395
+ f"{_DEPS_MISSING}"}
396
+ if not modality_chips:
397
+ return {"ok": False, "err": "no modality chips supplied"}
398
+ t0 = time.time()
399
+ try:
400
+ task = _ensure_adapter(adapter_name)
401
+ spec = ADAPTER_SPECS[adapter_name]
402
+ # Strip out bounds_4326 (auxiliary metadata, not a tensor) before
403
+ # handing the dict to terratorch's tiled_inference, which iterates
404
+ # all values as modalities.
405
+ tensors_only = {k: v for k, v in modality_chips.items()
406
+ if k != "bounds_4326"}
407
+ logits = _tiled_predict(task, tensors_only, spec["num_classes"])
408
+ # logits: (B, C, H, W). Argmax to per-pixel class id.
409
+ pred = logits.argmax(dim=1).squeeze(0)
410
+ result = summarizer(pred, spec["class_labels"])
411
+ result["elapsed_s"] = round(time.time() - t0, 2)
412
+ result["adapter"] = adapter_name
413
+ result["repo"] = ADAPTERS_REPO
414
+ result["compute"] = "local"
415
+ return result
416
+ except Exception as e:
417
+ msg = str(e)
418
+ # Translate torchvision binary-extension failures into a clean
419
+ # skip. terratorch + torchvision both ride a transitive
420
+ # dep cone on the HF Space (sentence-transformers pulls torch
421
+ # CPU; torchvision's C extension can't load against that wheel),
422
+ # so a local _ensure_adapter() raises RuntimeError with this
423
+ # signature when remote is also unreachable. Clean skip is the
424
+ # honest demo outcome β€” same as terramind_synthesis.
425
+ if "torchvision::nms" in msg or "torchvision_C" in msg:
426
+ log.warning("terramind_nyc/%s: torchvision binary unavailable; "
427
+ "remote unreachable too; clean skip", adapter_name)
428
+ return {"ok": False,
429
+ "skipped": "remote inference unreachable + local "
430
+ "torchvision binary unavailable on this "
431
+ "deployment",
432
+ "elapsed_s": round(time.time() - t0, 2)}
433
+ log.exception("terramind_nyc.%s failed", adapter_name)
434
+ return {"ok": False, "err": f"{type(e).__name__}: {e}",
435
+ "elapsed_s": round(time.time() - t0, 2)}
436
+
437
+
438
+ def lulc(s2l2a, s1rtc=None, dem=None,
439
+ bounds_4326: tuple[float, float, float, float] | None = None,
440
+ ) -> dict[str, Any]:
441
+ """5-class NYC macro land-cover.
442
+
443
+ Inputs are torch tensors. The temporal models we trained expect
444
+ [C, T, H, W] (preferred) or [C, H, W] (will be expanded to T=1).
445
+ Pass S1 and DEM if you have them β€” the published adapter was
446
+ trained on the full triplet and accuracy degrades when modalities
447
+ are dropped.
448
+
449
+ `bounds_4326` is `(minlon, minlat, maxlon, maxlat)` of the chip
450
+ in WGS84; when provided, the LULC raster is polygonised onto the
451
+ chip's geographic extent so the map can render an overlay.
452
+ """
453
+ chips = {"S2L2A": s2l2a}
454
+ if bounds_4326 is not None:
455
+ chips["bounds_4326"] = bounds_4326
456
+ if s1rtc is not None:
457
+ chips["S1RTC"] = s1rtc
458
+ if dem is not None:
459
+ chips["DEM"] = dem
460
+ return _run("lulc", chips, _summarize_lulc)
461
+
462
+
463
+ def buildings(s2l2a, s1rtc=None, dem=None,
464
+ bounds_4326: tuple[float, float, float, float] | None = None,
465
+ ) -> dict[str, Any]:
466
+ """Binary NYC building-footprint mask. Same input contract as lulc()."""
467
+ chips = {"S2L2A": s2l2a}
468
+ if bounds_4326 is not None:
469
+ chips["bounds_4326"] = bounds_4326
470
+ if s1rtc is not None:
471
+ chips["S1RTC"] = s1rtc
472
+ if dem is not None:
473
+ chips["DEM"] = dem
474
+ return _run("buildings", chips, _summarize_buildings)
475
+
476
+
477
+ def warm():
478
+ """Optional pre-load β€” amortizes the first-query model build cost."""
479
+ if not ENABLE or not _DEPS_OK:
480
+ return
481
+ try:
482
+ for name in ADAPTER_SPECS:
483
+ _ensure_adapter(name)
484
+ except Exception:
485
+ log.exception("terramind_nyc: warm() failed; specialists will no-op")
app/context/terramind_synthesis.py ADDED
@@ -0,0 +1,468 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """TerraMind v1 base as a real-time FSM node β€” DEM β†’ ESRI LULC.
2
+
3
+ Per user query: take the geocoded (lat, lon), pull a DEM patch from
4
+ Riprap's existing NYC-wide LiDAR raster (already used by the microtopo
5
+ specialist β€” no STAC dependency), run TerraMind to generate a
6
+ plausible categorical land-cover map from the terrain context, and
7
+ emit class fractions the reconciler can cite as a synthetic-prior
8
+ context layer alongside the empirical and modeled flood evidence.
9
+
10
+ Why DEM β†’ LULC (and not DEM β†’ S2L2A as initially prototyped):
11
+ - LULC is *categorical* and *interpretable*. The output is one of
12
+ 10 ESRI Land Cover classes per pixel; class fractions like "78%
13
+ Built Area" go straight into the briefing as cite-able claims.
14
+ - S2L2A is 12-channel reflectance β€” uninterpretable downstream
15
+ without a separate segmentation head.
16
+ - LULC is *comparable to ground truth*: NYC PLUTO land-use class
17
+ is already in the data layer; future calibration possible.
18
+
19
+ Class label mapping is *tentative* against ESRI 2020-2022 schema
20
+ (which TerraMesh's LULC tokenizer was trained on). The doc body
21
+ discloses the mapping as tentative and the reconciler is instructed
22
+ to use hedged framing ("the synthetic land-cover prior identifies …
23
+ likely class …") rather than asserting hard labels.
24
+
25
+ Why this shape:
26
+ - **No STAC dependency.** Microsoft Planetary Computer search has
27
+ been intermittent during this hackathon; the DEM raster is local
28
+ and always available.
29
+ - **Real-time.** < 0.3 s synthesis + < 0.5 s DEM patch read on M3
30
+ CPU once warm.
31
+ - **Honesty discipline.** Synthetic-prior tier, fourth epistemic
32
+ class alongside empirical / modeled / proxy.
33
+
34
+ License: Apache-2.0 β€” `ibm-esa-geospatial/TerraMind-1.0-base`.
35
+ """
36
+
37
+ from __future__ import annotations
38
+
39
+ import logging
40
+ import os
41
+ import random
42
+ import threading
43
+ import time
44
+ from typing import Any
45
+
46
+ log = logging.getLogger("riprap.terramind")
47
+
48
+ ENABLE = os.environ.get("RIPRAP_TERRAMIND_ENABLE", "1").lower() in ("1", "true", "yes")
49
+ DEFAULT_STEPS = int(os.environ.get("RIPRAP_TERRAMIND_STEPS", "10"))
50
+ DEFAULT_SEED = int(os.environ.get("RIPRAP_TERRAMIND_SEED", "42"))
51
+ CHIP_PX = int(os.environ.get("RIPRAP_TERRAMIND_CHIP_PX", "224"))
52
+ CHIP_M = CHIP_PX * 30 # NYC DEM is at 30 m -> 6.72 km square
53
+ HALF_M = CHIP_M / 2
54
+
55
+ _MODEL = None
56
+ _INIT_LOCK = threading.Lock()
57
+
58
+ # Tentative ESRI 2020-2022 Land Cover class mapping for TerraMind v1's
59
+ # LULC tokenizer output (10 channels, argmax over channel axis -> class
60
+ # index 0-9). The README/docs don't expose the exact mapping and the
61
+ # tokenizer source confirms only "ESRI LULC" without a label table, so
62
+ # the names below are best-effort. The doc body discloses tentativeness.
63
+ LULC_CLASSES = [
64
+ "water", # 0
65
+ "trees", # 1
66
+ "grass", # 2
67
+ "flooded_vegetation", # 3
68
+ "crops", # 4
69
+ "scrub_shrub", # 5
70
+ "built_area", # 6
71
+ "bare_ground", # 7
72
+ "snow_ice", # 8
73
+ "clouds_or_no_data", # 9
74
+ ]
75
+
76
+
77
+ def _has_required_deps() -> tuple[bool, str | None]:
78
+ """Probe deps. terramind_synthesis runs only locally (no remote path
79
+ in app/inference.py for DEM-driven synthesis), so it always needs
80
+ terratorch. On the HF Space terratorch isn't installed, so this
81
+ specialist returns a clean `skipped: deps unavailable` outcome.
82
+
83
+ Distinguishes a *truly missing* package (ModuleNotFoundError) from
84
+ a *transient race* (other ImportError β€” typically sklearn's
85
+ "partially initialized module" from concurrent imports)."""
86
+ missing = []
87
+ for name in ("terratorch", "rasterio"):
88
+ try:
89
+ __import__(name)
90
+ except ModuleNotFoundError:
91
+ missing.append(name)
92
+ except ImportError:
93
+ log.debug("terramind: import race on %s, will retry on demand", name)
94
+ except Exception as e:
95
+ # torchvision::nms RuntimeError on HF Space β€” local inference
96
+ # is unavailable; treat as missing so fetch() returns a clean
97
+ # skip rather than crashing in _ensure_model.
98
+ log.warning("terramind: %s import raised %s; treating as "
99
+ "unavailable", name, type(e).__name__)
100
+ missing.append(f"{name} ({type(e).__name__})")
101
+ return (not missing, ", ".join(missing) if missing else None)
102
+
103
+
104
+ _DEPS_OK, _DEPS_MISSING = _has_required_deps()
105
+
106
+
107
+ def _ensure_model():
108
+ """Lazy load with a lock so the parallel-block worker can't double-init."""
109
+ global _MODEL
110
+ if _MODEL is not None:
111
+ return _MODEL
112
+ with _INIT_LOCK:
113
+ if _MODEL is not None:
114
+ return _MODEL
115
+ # Heavy import deferred to first call so module import stays cheap
116
+ # and HF Spaces (no terratorch) doesn't pay it at all.
117
+ import terratorch.models.backbones.terramind.model.terramind_register # noqa
118
+ from terratorch.registry import FULL_MODEL_REGISTRY
119
+ log.info("terramind: loading v1 base generate (DEM -> LULC)")
120
+ m = FULL_MODEL_REGISTRY.build(
121
+ "terratorch_terramind_v1_base_generate",
122
+ modalities=["DEM"],
123
+ output_modalities=["LULC"],
124
+ pretrained=True,
125
+ timesteps=DEFAULT_STEPS,
126
+ )
127
+ m.eval()
128
+ _MODEL = m
129
+ log.info("terramind: model ready")
130
+ return _MODEL
131
+
132
+
133
+ def warm():
134
+ """Call at app boot to amortize the ~6 s checkpoint load + first-call
135
+ JIT. No-op when deps are absent."""
136
+ if ENABLE and _DEPS_OK:
137
+ try:
138
+ _ensure_model()
139
+ except Exception:
140
+ log.exception("terramind: warm() failed; specialist will no-op")
141
+
142
+
143
+ def _read_dem_patch(lat: float, lon: float):
144
+ """Read a CHIP_PXΓ—CHIP_PX DEM patch centered on (lat, lon) from the
145
+ local NYC-wide LiDAR raster. Returns (array, bounds_4326) where
146
+ bounds_4326 is (minlon, minlat, maxlon, maxlat) so the synthesised
147
+ LULC can be georeferenced onto the same extent for map rendering.
148
+ Returns None if outside the raster's extent."""
149
+ from pathlib import Path
150
+
151
+ import numpy as np
152
+ import rasterio
153
+ from rasterio.windows import from_bounds
154
+ dem_path = (Path(__file__).resolve().parents[2]
155
+ / "data" / "nyc_dem_30m.tif")
156
+ if not dem_path.exists():
157
+ return None
158
+ with rasterio.open(dem_path) as src:
159
+ # The DEM is in EPSG:4326 (geographic) in our cache β€” convert
160
+ # the chip extent in the same CRS by building a rough degree
161
+ # bbox from a meters-square half-side at NYC latitude.
162
+ # 1 degree lat β‰ˆ 111 km, 1 degree lon β‰ˆ 85 km at 40.7Β°N.
163
+ d_lat = (HALF_M / 111_000.0)
164
+ d_lon = (HALF_M / 85_000.0)
165
+ win = from_bounds(lon - d_lon, lat - d_lat,
166
+ lon + d_lon, lat + d_lat,
167
+ src.transform)
168
+ arr = src.read(1, window=win, boundless=True, fill_value=0).astype("float32")
169
+ if arr.size == 0 or arr.shape[0] < 8 or arr.shape[1] < 8:
170
+ return None
171
+ # Resize to CHIP_PX Γ— CHIP_PX via torch interpolation. The exact
172
+ # pixel-perfect alignment doesn't matter for a synthetic prior; the
173
+ # model just needs a real terrain patch to condition on.
174
+ import torch
175
+ t = torch.from_numpy(arr).unsqueeze(0).unsqueeze(0)
176
+ t = torch.nn.functional.interpolate(t, size=(CHIP_PX, CHIP_PX),
177
+ mode="bilinear", align_corners=False)
178
+ out = t.squeeze(0).numpy() # (1, CHIP_PX, CHIP_PX)
179
+ # Replace NaN sentinel values with median elevation so the model
180
+ # doesn't see NaN tokens.
181
+ if np.isnan(out).any():
182
+ med = float(np.nanmedian(out))
183
+ out = np.nan_to_num(out, nan=med)
184
+ bounds_4326 = (lon - d_lon, lat - d_lat, lon + d_lon, lat + d_lat)
185
+ return out, bounds_4326
186
+
187
+
188
+ # Map class index -> visual color for the categorical fill on the
189
+ # MapLibre layer. Colors picked to be visually distinct from the
190
+ # existing red (Sandy) / blue (DEP) / cyan (Prithvi) / orange (Ida HWM).
191
+ LULC_FILL_COLORS = {
192
+ "water": "#0284c7", # not used (we keep water clear so
193
+ # the underlying basemap shows)
194
+ "trees": "#16a34a", # green
195
+ "grass": "#86efac", # pale green
196
+ "flooded_vegetation": "#a3e635", # lime
197
+ "crops": "#fde047", # yellow
198
+ "scrub_shrub": "#bef264",
199
+ "built_area": "#9ca3af", # neutral gray
200
+ "bare_ground": "#d6d3d1", # warm light gray
201
+ "snow_ice": "#f3f4f6",
202
+ "clouds_or_no_data": "#000000", # not used (kept transparent)
203
+ }
204
+ # Classes we don't render at all (transparent) β€” water is best left
205
+ # uncolored so the basemap shoreline reads through; clouds/no-data is
206
+ # semantically meaningless to fill.
207
+ LULC_HIDE_CLASSES = {"water", "clouds_or_no_data"}
208
+
209
+
210
+ def _polygonize_lulc(class_idx, bounds_4326: tuple) -> dict:
211
+ """Vectorize the per-pixel argmax classification into one MultiPolygon
212
+ per class label, then dump as a single GeoJSON FeatureCollection in
213
+ EPSG:4326. Each feature carries `label` + `class_idx` properties so
214
+ the frontend can colour by category.
215
+ """
216
+ import json
217
+
218
+ import geopandas as gpd
219
+ from rasterio.features import shapes
220
+ from rasterio.transform import from_bounds as transform_from_bounds
221
+ from shapely.geometry import shape
222
+
223
+ minlon, minlat, maxlon, maxlat = bounds_4326
224
+ h, w = class_idx.shape
225
+ transform = transform_from_bounds(minlon, minlat, maxlon, maxlat, w, h)
226
+ feats = []
227
+ for i, label in enumerate(LULC_CLASSES):
228
+ if label in LULC_HIDE_CLASSES:
229
+ continue
230
+ mask = (class_idx == i).astype("uint8")
231
+ if mask.sum() < 8: # skip tiny noise
232
+ continue
233
+ polys = []
234
+ for geom, value in shapes(mask, mask=mask.astype(bool),
235
+ transform=transform):
236
+ if value != 1:
237
+ continue
238
+ polys.append(shape(geom))
239
+ if not polys:
240
+ continue
241
+ # Dissolve via geopandas + simplify lightly. The chip is 30 m
242
+ # per pixel and we don't need pixel-edge fidelity at urban zoom.
243
+ gdf = gpd.GeoDataFrame({"geometry": polys}, crs="EPSG:4326")
244
+ gdf["geometry"] = gdf.geometry.simplify(1e-4, preserve_topology=True)
245
+ for geom in gdf.geometry:
246
+ feats.append({
247
+ "type": "Feature",
248
+ "geometry": json.loads(gpd.GeoSeries([geom],
249
+ crs="EPSG:4326").to_json())["features"][0]["geometry"],
250
+ "properties": {"label": label, "class_idx": i,
251
+ "fill_color": LULC_FILL_COLORS.get(label, "#9ca3af")},
252
+ })
253
+ return {"type": "FeatureCollection", "features": feats}
254
+
255
+
256
+ def fetch(lat: float, lon: float, timeout_s: float = 60.0) -> dict[str, Any]:
257
+ """Run the specialist. Returns:
258
+ { ok: bool,
259
+ skipped: str | None,
260
+ synthetic_modality: bool,
261
+ tim_chain: list[str],
262
+ diffusion_steps: int, diffusion_seed: int,
263
+ dem_mean_m: float,
264
+ class_fractions: dict[str, float], # tentative ESRI labels
265
+ dominant_class: str, # highest-fraction label
266
+ dominant_pct: float,
267
+ n_classes_observed: int,
268
+ chip_shape: list[int],
269
+ elapsed_s: float,
270
+ err: str | None }
271
+
272
+ Designed never to raise. Failures show up as ok=False with reason.
273
+ """
274
+ if not ENABLE:
275
+ return {"ok": False, "skipped": "RIPRAP_TERRAMIND_ENABLE=0"}
276
+ t0 = time.time()
277
+ try:
278
+ import numpy as np
279
+ patch = _read_dem_patch(lat, lon)
280
+ if patch is None:
281
+ return {"ok": False, "skipped": "no DEM coverage at this point"}
282
+ dem, bounds_4326 = patch
283
+ dem_mean = float(dem.mean())
284
+
285
+ # v0.4.5+ β€” try the MI300X inference service first if configured.
286
+ # The droplet's /v1/terramind dispatch handles adapter='synthesis'
287
+ # via _terramind_synthesis_inference (DEM -> generative LULC). On
288
+ # the HF Space terratorch's torchvision binary doesn't load, so
289
+ # this is the only working path there.
290
+ try:
291
+ from app import inference as _inf
292
+ if _inf.remote_enabled():
293
+ # The terramind v1 base generative encoder embedding
294
+ # layer unpacks `B, C, H, W = x.shape` (verified against
295
+ # terratorch_terramind_v1_base_generate). DEM has C=1, so
296
+ # the on-the-wire shape is (1, 1, H, W) 4-D.
297
+ # `_read_dem_patch` returns a 3-D (1, H, W) array (it
298
+ # interpolates to CHIP_PXΓ—CHIP_PX through a 4-D
299
+ # torch.functional.interpolate then squeezes the batch),
300
+ # so we add only the batch dim β€” not two.
301
+ import numpy as _np_local
302
+ dem_arr = _np_local.asarray(dem, dtype="float32")
303
+ if dem_arr.ndim == 2: # (H, W)
304
+ dem_remote = dem_arr[None, None, :, :]
305
+ elif dem_arr.ndim == 3: # (1, H, W)
306
+ dem_remote = dem_arr[None, :, :, :]
307
+ elif dem_arr.ndim == 4: # already (1, 1, H, W)
308
+ dem_remote = dem_arr
309
+ else:
310
+ raise ValueError(
311
+ f"unexpected DEM shape {dem_arr.shape}; "
312
+ "expected 2/3/4-D")
313
+ remote = _inf.terramind("synthesis", None, None, dem_remote,
314
+ timeout=timeout_s)
315
+ if remote.get("ok"):
316
+ elapsed = round(time.time() - t0, 2)
317
+ # Polygonize the prediction raster for the map
318
+ # layer. The droplet returns the per-pixel argmax;
319
+ # we vectorize against the chip's bounds.
320
+ polys = None
321
+ pred_b64 = remote.get("pred_b64")
322
+ pred_shape = remote.get("pred_shape")
323
+ class_labels = (remote.get("class_labels")
324
+ or LULC_CLASSES)
325
+ if pred_b64 and pred_shape:
326
+ try:
327
+ from app.context._polygonize import (
328
+ polygonize_class_raster,
329
+ )
330
+ polys = polygonize_class_raster(
331
+ pred_b64, pred_shape, class_labels,
332
+ tuple(bounds_4326),
333
+ simplify_tolerance=2e-5,
334
+ )
335
+ except Exception:
336
+ log.exception("terramind/synthesis: "
337
+ "polygonize failed")
338
+ polys = None
339
+ out = {
340
+ "ok": True,
341
+ "synthetic_modality": True,
342
+ "tim_chain": ["DEM", "LULC_synthetic"],
343
+ "diffusion_steps": remote.get("diffusion_steps",
344
+ DEFAULT_STEPS),
345
+ "diffusion_seed": DEFAULT_SEED,
346
+ "dem_mean_m": round(dem_mean, 2),
347
+ "class_fractions": remote.get("class_fractions") or {},
348
+ "dominant_class": remote.get("dominant_class") or "unknown",
349
+ "dominant_pct": remote.get("dominant_pct") or 0.0,
350
+ "n_classes_observed": remote.get("n_classes_observed") or 0,
351
+ "chip_shape": remote.get("shape") or [],
352
+ "bounds_4326": list(bounds_4326),
353
+ "polygons_geojson": polys,
354
+ "label_schema": remote.get("label_schema") or "",
355
+ "compute": f"remote Β· {remote.get('device', 'gpu')}",
356
+ "elapsed_s": elapsed,
357
+ }
358
+ return out
359
+ # remote returned non-ok β€” surface that signal directly
360
+ return {"ok": False,
361
+ "skipped": f"remote terramind synthesis non-ok: "
362
+ f"{remote.get('error') or remote.get('detail') or 'unknown'}",
363
+ "elapsed_s": round(time.time() - t0, 2)}
364
+ except _inf.RemoteUnreachable as e:
365
+ log.info("terramind_synthesis: remote unreachable (%s); local fallback", e)
366
+ except Exception as e:
367
+ log.exception("terramind_synthesis: remote call failed")
368
+ return {"ok": False,
369
+ "skipped": f"remote terramind synthesis error: "
370
+ f"{type(e).__name__}: {e}",
371
+ "elapsed_s": round(time.time() - t0, 2)}
372
+
373
+ # Local fallback β€” original path; only available where terratorch
374
+ # imports without the torchvision::nms RuntimeError.
375
+ if not _DEPS_OK:
376
+ return {"ok": False, "skipped": f"deps unavailable: {_DEPS_MISSING}"}
377
+ import torch
378
+ random.seed(DEFAULT_SEED)
379
+ torch.manual_seed(DEFAULT_SEED)
380
+
381
+ model = _ensure_model()
382
+ # `dem` is 2-D (H, W) from `_read_dem_patch.src.read(1, ...)`. The
383
+ # terramind v1 base generative encoder wants (B=1, C=1, H, W) 4-D.
384
+ dem_t = torch.from_numpy(dem).unsqueeze(0).unsqueeze(0).float()
385
+ if time.time() - t0 > timeout_s:
386
+ return {"ok": False, "skipped": "terramind exceeded budget"}
387
+
388
+ with torch.no_grad():
389
+ out = model({"DEM": dem_t}, timesteps=DEFAULT_STEPS,
390
+ verbose=False)
391
+ lulc = out["LULC"]
392
+ if hasattr(lulc, "detach"):
393
+ lulc = lulc.detach().cpu().numpy()
394
+ if lulc.ndim == 4:
395
+ lulc = lulc[0] # (n_classes, H, W)
396
+ # Argmax over class channel -> per-pixel class index, then
397
+ # fraction by class. This is the cite-able structured output.
398
+ class_idx = lulc.argmax(axis=0) # (H, W)
399
+ unique, counts = np.unique(class_idx, return_counts=True)
400
+ total = float(class_idx.size)
401
+ fractions: dict[str, float] = {}
402
+ for u, c in zip(unique, counts, strict=False):
403
+ label = (LULC_CLASSES[int(u)] if 0 <= int(u) < len(LULC_CLASSES)
404
+ else f"class_{int(u)}")
405
+ fractions[label] = round(100.0 * c / total, 2)
406
+ # Sort dominant -> tail for deterministic doc body ordering.
407
+ ordered = dict(sorted(fractions.items(),
408
+ key=lambda kv: kv[1], reverse=True))
409
+ dominant_class = next(iter(ordered)) if ordered else "unknown"
410
+ dominant_pct = ordered.get(dominant_class, 0.0)
411
+ # Class indices map to TerraMesh's LULC tokenizer codebook; the
412
+ # exact label-to-index mapping isn't published. Surface a tentative
413
+ # name plus the raw index so a reader can see we're not asserting
414
+ # ground truth.
415
+ dominant_idx = next((i for i, lbl in enumerate(LULC_CLASSES)
416
+ if lbl == dominant_class), -1)
417
+ dominant_display = (
418
+ f"class_{dominant_idx} (tentative: {dominant_class})"
419
+ if dominant_idx >= 0 else dominant_class
420
+ )
421
+
422
+ # Polygonize the categorical raster for the map layer.
423
+ # Best-effort β€” failure here doesn't fail the specialist.
424
+ try:
425
+ polygons_geojson = _polygonize_lulc(class_idx, bounds_4326)
426
+ except Exception:
427
+ log.exception("terramind: polygonize failed; skipping map layer")
428
+ polygons_geojson = None
429
+
430
+ return {
431
+ "ok": True,
432
+ "synthetic_modality": True,
433
+ "tim_chain": ["DEM", "LULC_synthetic"],
434
+ "diffusion_steps": DEFAULT_STEPS,
435
+ "diffusion_seed": DEFAULT_SEED,
436
+ "dem_mean_m": round(dem_mean, 2),
437
+ "class_fractions": ordered,
438
+ "dominant_class": dominant_class,
439
+ "dominant_class_display": dominant_display,
440
+ "dominant_pct": dominant_pct,
441
+ "n_classes_observed": len(ordered),
442
+ "chip_shape": list(lulc.shape),
443
+ "bounds_4326": list(bounds_4326),
444
+ "polygons_geojson": polygons_geojson,
445
+ "label_schema": "ESRI 2020-2022 Land Cover (tentative β€” "
446
+ "TerraMind tokenizer source confirms ESRI but "
447
+ "not exact label-to-index mapping)",
448
+ "elapsed_s": round(time.time() - t0, 2),
449
+ }
450
+ except Exception as e:
451
+ msg = str(e)
452
+ # Translate the torchvision binary-extension failure into a clean
453
+ # skip. The HF Space ships torchvision via a transitive sentence-
454
+ # transformers dep, but its C extension can't load alongside our
455
+ # CPU torch wheel, so terratorch's NMS call raises RuntimeError.
456
+ # Surface this honestly β€” the local inference path is unavailable
457
+ # on this deployment, same outcome as a missing terratorch.
458
+ if "torchvision::nms" in msg or "torchvision_C" in msg:
459
+ log.warning("terramind: torchvision binary unavailable on this "
460
+ "deployment; skipping local inference")
461
+ return {"ok": False,
462
+ "skipped": "local inference unavailable on this "
463
+ "deployment (torchvision binary extension "
464
+ "not loadable); no remote synthesis path",
465
+ "elapsed_s": round(time.time() - t0, 2)}
466
+ log.exception("terramind: fetch failed")
467
+ return {"ok": False, "err": f"{type(e).__name__}: {e}",
468
+ "elapsed_s": round(time.time() - t0, 2)}
app/emissions.py ADDED
@@ -0,0 +1,269 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Per-query emissions tracker for inference calls.
2
+
3
+ Records every LLM and ML-inference call made during a single query and
4
+ summarizes:
5
+ - wallclock duration per call
6
+ - prompt + completion tokens (LLM)
7
+ - energy in watt-hours, **measured from the L4 GPU when available**
8
+ (the inference proxy reports per-call `X-GPU-Power-W` /
9
+ `X-GPU-Energy-J` headers from a 100 ms-cadence NVML sampler).
10
+ Falls back to a duration Γ— data-sheet-power estimate when the
11
+ proxy is unreachable / NVML init failed / call went to a backend
12
+ that doesn't surface power readings.
13
+
14
+ Each call record carries a `measured: bool` flag indicating which path
15
+ was used, so the UI can disclose. `summarize()` aggregates total Wh,
16
+ total tokens, by-kind and by-hardware splits β€” no cloud comparison.
17
+
18
+ Thread propagation
19
+ ------------------
20
+ The tracker is held in a thread-local. The dispatch layer
21
+ (web/main.py) installs one per request; `app/fsm.py:iter_steps`
22
+ captures and re-installs it on the FSM runner thread (mirroring the
23
+ existing `_captured_token_cb` pattern). Worker threads spawned inside
24
+ specialists (prithvi_live, eo_chip_cache) inherit nothing β€” those calls
25
+ are silently dropped, which is acceptable: those specialists do <1 s of
26
+ inference each and are off the hot path for the energy story.
27
+ """
28
+ from __future__ import annotations
29
+
30
+ import threading
31
+ from typing import Any
32
+
33
+ # (label, fallback_sustained_power_w, source). Used only when the
34
+ # proxy doesn't surface a real measurement (NVML disabled, backend
35
+ # unreachable, local-fallback path). The fallback figure is a
36
+ # conservative public-record estimate; the `measured: bool` flag on
37
+ # each call record indicates whether the row used the fallback.
38
+ HARDWARE: dict[str, tuple[str, float, str]] = {
39
+ "nvidia_l4": (
40
+ "NVIDIA L4",
41
+ 60.0,
42
+ "NVIDIA L4 Tensor Core GPU data sheet (72 W TGP, Ada Lovelace, "
43
+ "24 GB); ~60 W sustained during transformer inference. The "
44
+ "active backend for the Riprap inference Space "
45
+ "(msradam/riprap-vllm). When the proxy is reachable and NVML "
46
+ "is initialized, real per-call power is read off the device "
47
+ "via nvmlDeviceGetPowerUsage and this fallback is unused.",
48
+ ),
49
+ "amd_mi300x": (
50
+ "AMD MI300X",
51
+ 600.0,
52
+ "AMD Instinct MI300X data sheet (750 W TDP); ~600 W sustained "
53
+ "during vLLM generation. Selected only when an operator deploys "
54
+ "against an MI300X droplet and sets RIPRAP_HARDWARE_LABEL=AMD "
55
+ "MI300X explicitly. The hackathon submission used to run on "
56
+ "this hardware; the droplet was decommissioned 2026-05-06.",
57
+ ),
58
+ "nvidia_t4": (
59
+ "NVIDIA T4",
60
+ 50.0,
61
+ "NVIDIA T4 data sheet (70 W max); ~50 W sustained during "
62
+ "transformer inference.",
63
+ ),
64
+ "apple_m": (
65
+ "Apple M-series",
66
+ 20.0,
67
+ "ml.energy / community measurements: ~20 W package power "
68
+ "during Granite 4.1 q4_K_M inference on Apple M3/M4 (the "
69
+ "local-dev path, no remote backend configured).",
70
+ ),
71
+ "cpu_server": (
72
+ "x86 CPU",
73
+ 30.0,
74
+ "Typical sustained x86 server-core load (~30 W) for CPU-only "
75
+ "inference fallbacks.",
76
+ ),
77
+ }
78
+
79
+
80
+ def _wh(power_w: float, duration_s: float) -> float:
81
+ return power_w * max(duration_s, 0.0) / 3600.0
82
+
83
+
84
+ class Tracker:
85
+ """Append-only call ledger for one query. Thread-safe."""
86
+
87
+ def __init__(self) -> None:
88
+ self.calls: list[dict[str, Any]] = []
89
+ self._lock = threading.Lock()
90
+
91
+ def _record(self, *, base: dict[str, Any], hardware: str,
92
+ duration_s: float,
93
+ joules_real: float | None,
94
+ power_w_real: float | None) -> None:
95
+ """Shared body of record_llm / record_ml.
96
+
97
+ When `joules_real` is provided (NVML-derived from the proxy),
98
+ we use it directly and stamp `measured=True`. Otherwise we
99
+ fall back to the data-sheet sustained-power estimate.
100
+ """
101
+ hw_label, fallback_w, _src = HARDWARE.get(hardware,
102
+ HARDWARE["cpu_server"])
103
+ if joules_real is not None and joules_real >= 0:
104
+ joules = float(joules_real)
105
+ wh = joules / 3600.0
106
+ measured = True
107
+ avg_w = (joules / duration_s) if duration_s > 0 else (
108
+ power_w_real if power_w_real is not None else fallback_w)
109
+ else:
110
+ avg_w = fallback_w
111
+ wh = _wh(avg_w, duration_s)
112
+ joules = wh * 3600.0
113
+ measured = False
114
+ record = {
115
+ **base,
116
+ "hardware": hardware,
117
+ "hardware_label": hw_label,
118
+ "power_w": round(avg_w, 2),
119
+ "duration_s": round(duration_s, 3),
120
+ "measured": measured,
121
+ "wh": round(wh, 5),
122
+ "joules": round(joules, 3),
123
+ }
124
+ with self._lock:
125
+ self.calls.append(record)
126
+
127
+ def record_llm(self, *, model: str, backend: str, hardware: str,
128
+ prompt_tokens: int | None,
129
+ completion_tokens: int | None,
130
+ duration_s: float,
131
+ stream: bool = False,
132
+ joules_real: float | None = None,
133
+ power_w_real: float | None = None) -> None:
134
+ total = None
135
+ if prompt_tokens is not None or completion_tokens is not None:
136
+ total = (prompt_tokens or 0) + (completion_tokens or 0)
137
+ self._record(
138
+ base={
139
+ "kind": "llm",
140
+ "model": model,
141
+ "backend": backend,
142
+ "prompt_tokens": prompt_tokens,
143
+ "completion_tokens": completion_tokens,
144
+ "total_tokens": total,
145
+ "stream": stream,
146
+ },
147
+ hardware=hardware,
148
+ duration_s=duration_s,
149
+ joules_real=joules_real,
150
+ power_w_real=power_w_real,
151
+ )
152
+
153
+ def record_ml(self, *, endpoint: str, backend: str, hardware: str,
154
+ duration_s: float,
155
+ joules_real: float | None = None,
156
+ power_w_real: float | None = None) -> None:
157
+ self._record(
158
+ base={
159
+ "kind": "ml",
160
+ "endpoint": endpoint,
161
+ "backend": backend,
162
+ },
163
+ hardware=hardware,
164
+ duration_s=duration_s,
165
+ joules_real=joules_real,
166
+ power_w_real=power_w_real,
167
+ )
168
+
169
+ def summarize(self) -> dict[str, Any]:
170
+ with self._lock:
171
+ calls = list(self.calls)
172
+ total_wh = sum(c["wh"] for c in calls)
173
+ total_dur = sum(c["duration_s"] for c in calls)
174
+ n_measured = sum(1 for c in calls if c.get("measured"))
175
+ prompt = sum((c.get("prompt_tokens") or 0)
176
+ for c in calls if c["kind"] == "llm")
177
+ completion = sum((c.get("completion_tokens") or 0)
178
+ for c in calls if c["kind"] == "llm")
179
+
180
+ by_kind: dict[str, dict[str, Any]] = {}
181
+ for c in calls:
182
+ slot = by_kind.setdefault(c["kind"], {"wh": 0.0, "n": 0,
183
+ "duration_s": 0.0})
184
+ slot["wh"] += c["wh"]
185
+ slot["n"] += 1
186
+ slot["duration_s"] += c["duration_s"]
187
+ for slot in by_kind.values():
188
+ slot["wh"] = round(slot["wh"], 5)
189
+ slot["mwh"] = round(slot["wh"] * 1000, 2)
190
+ slot["duration_s"] = round(slot["duration_s"], 3)
191
+
192
+ by_hw: dict[str, dict[str, Any]] = {}
193
+ for c in calls:
194
+ slot = by_hw.setdefault(c["hardware"], {
195
+ "label": c["hardware_label"],
196
+ "wh": 0.0, "n": 0, "duration_s": 0.0,
197
+ })
198
+ slot["wh"] += c["wh"]
199
+ slot["n"] += 1
200
+ slot["duration_s"] += c["duration_s"]
201
+ for slot in by_hw.values():
202
+ slot["wh"] = round(slot["wh"], 5)
203
+ slot["mwh"] = round(slot["wh"] * 1000, 2)
204
+ slot["duration_s"] = round(slot["duration_s"], 3)
205
+
206
+ return {
207
+ "n_calls": len(calls),
208
+ "n_measured": n_measured,
209
+ "total_wh": round(total_wh, 5),
210
+ "total_mwh": round(total_wh * 1000, 2),
211
+ "total_joules": round(total_wh * 3600, 1),
212
+ "total_duration_s": round(total_dur, 3),
213
+ "tokens": {
214
+ "prompt": prompt or None,
215
+ "completion": completion or None,
216
+ "total": (prompt + completion) or None,
217
+ },
218
+ "by_kind": by_kind,
219
+ "by_hardware": by_hw,
220
+ "calls": calls,
221
+ "method": (
222
+ "Energy is read off the L4 GPU per call via "
223
+ "nvmlDeviceGetPowerUsage on the inference proxy "
224
+ "(X-GPU-Energy-J response header). Calls flagged "
225
+ "measured=false fall back to "
226
+ "(data-sheet sustained_power_w Γ— duration_s Γ· 3600) "
227
+ "β€” see app/emissions.HARDWARE for sources. Tokens "
228
+ "are reported by the backend (LiteLLM usage) when "
229
+ "available, else estimated from response text length "
230
+ "(~4 chars/token)."
231
+ ),
232
+ }
233
+
234
+
235
+ # Thread-local install. Calls made on threads without an installed
236
+ # tracker hit a no-op stub β€” always safe to call active().record_*().
237
+ _tl = threading.local()
238
+
239
+
240
+ class _NullTracker:
241
+ def record_llm(self, **_kw: Any) -> None:
242
+ return None
243
+
244
+ def record_ml(self, **_kw: Any) -> None:
245
+ return None
246
+
247
+
248
+ _NULL = _NullTracker()
249
+
250
+
251
+ def install(tracker: Tracker | None) -> None:
252
+ _tl.tracker = tracker
253
+
254
+
255
+ def current() -> Tracker | None:
256
+ return getattr(_tl, "tracker", None)
257
+
258
+
259
+ def active() -> Tracker | _NullTracker:
260
+ """Return the installed tracker for this thread, or a no-op stub.
261
+ Always safe to call in instrumentation hot paths."""
262
+ return getattr(_tl, "tracker", None) or _NULL
263
+
264
+
265
+ def estimate_completion_tokens(text: str) -> int:
266
+ """Rough char/4 estimator used when the backend doesn't report usage
267
+ (e.g. streaming through Ollama, where LiteLLM's stream wrapper does
268
+ not always surface a final usage block)."""
269
+ return max(1, len(text) // 4)
app/energy.py ADDED
@@ -0,0 +1,56 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Per-query energy footprint estimate.
2
+
3
+ Conservative, defensible numbers β€” no overclaim. We measure local
4
+ inference time and apply a published-range package-power figure for
5
+ Apple-Silicon LLM inference; we compare to the most recent published
6
+ estimate of frontier-cloud per-query energy (Epoch AI, 2025).
7
+
8
+ This is not a benchmark β€” it's a transparent rule-of-thumb that the
9
+ user can audit. The system prompt and the UI both surface the
10
+ underlying numbers and the citation.
11
+ """
12
+ from __future__ import annotations
13
+
14
+ # Local: Granite 4.1:3b on Apple M-series (M3/M4 Pro range)
15
+ # Sustained package power during ~5 s of LLM inference, q4_K_M quant.
16
+ # Source: ml.energy + community measurements; conservative midpoint.
17
+ LOCAL_PACKAGE_POWER_W = 20.0
18
+
19
+ # Frontier cloud per-query inference energy.
20
+ # Source: Epoch AI, "How much energy does ChatGPT use?" (2025).
21
+ # https://epoch.ai/gradient-updates/how-much-energy-does-chatgpt-use
22
+ # This is a typical-query estimate for GPT-4o-class inference; long-context
23
+ # queries scale roughly linearly with token count.
24
+ CLOUD_PER_QUERY_WH = 0.30
25
+
26
+ # Citation strings used in the UI.
27
+ LOCAL_SOURCE = ("ml.energy / community measurements; ~20 W package power "
28
+ "during Granite 4.1:3b q4_K_M inference on Apple M-series.")
29
+ CLOUD_SOURCE = ('Epoch AI (2025), "How much energy does ChatGPT use?", '
30
+ "estimating ~0.3 Wh per typical GPT-4o query.")
31
+
32
+
33
+ def estimate(reconcile_seconds: float, total_seconds: float | None = None) -> dict:
34
+ """Return a per-query energy estimate.
35
+
36
+ Args:
37
+ reconcile_seconds: wallclock of the Granite reconcile step (the
38
+ only step that meaningfully draws CPU/GPU power).
39
+ total_seconds: optional full-FSM wallclock for context.
40
+ """
41
+ local_wh = LOCAL_PACKAGE_POWER_W * reconcile_seconds / 3600.0
42
+ return {
43
+ "local_wh": round(local_wh, 4),
44
+ "local_mwh": round(local_wh * 1000, 1),
45
+ "cloud_wh": CLOUD_PER_QUERY_WH,
46
+ "cloud_mwh": round(CLOUD_PER_QUERY_WH * 1000, 1),
47
+ "ratio_cloud_over_local": round(CLOUD_PER_QUERY_WH / local_wh, 1) if local_wh > 0 else None,
48
+ "method": {
49
+ "local": f"{LOCAL_PACKAGE_POWER_W} W Γ— {reconcile_seconds:.2f} s Γ· 3600",
50
+ "local_source": LOCAL_SOURCE,
51
+ "cloud": f"{CLOUD_PER_QUERY_WH} Wh per query (published estimate)",
52
+ "cloud_source": CLOUD_SOURCE,
53
+ },
54
+ "reconcile_seconds": round(reconcile_seconds, 2),
55
+ "total_seconds": round(total_seconds, 2) if total_seconds is not None else None,
56
+ }
app/flood_layers/__init__.py ADDED
File without changes
app/flood_layers/dep_stormwater.py ADDED
@@ -0,0 +1,168 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """NYC DEP Stormwater Flood Maps β€” pluvial scenarios.
2
+
3
+ Four scenarios, all in EPSG:2263. Polygons are categorized by depth class:
4
+ 1 = Nuisance Flooding (>4" and ≀1 ft)
5
+ 2 = Deep and Contiguous Flooding (>1 ft and ≀4 ft)
6
+ 3 = Deep Contiguous Flooding (>4 ft)
7
+
8
+ Two query paths exist:
9
+ join_raster(point) β€” fast path. Samples the baked GeoTIFFs in
10
+ data/baked/. ~3 ms per scenario, ~70 ms cold-open. Used by
11
+ step_dep in the FSM.
12
+ join(assets) β€” legacy GDB path via gpd.sjoin. Retained as
13
+ a fallback when baked rasters are absent (local dev) and as
14
+ the polygon-overlap path used by coverage_for_polygon for
15
+ neighborhood mode.
16
+ """
17
+ from __future__ import annotations
18
+
19
+ import logging
20
+ import threading
21
+ from functools import lru_cache
22
+
23
+ import geopandas as gpd
24
+
25
+ from app.spatial import DATA, NYC_CRS
26
+
27
+ log = logging.getLogger(__name__)
28
+ BAKED = DATA / "baked"
29
+ _TLOCAL = threading.local()
30
+ _FALLBACK_WARNED = False
31
+
32
+ ROOT = DATA / "dep"
33
+
34
+ SCENARIOS = {
35
+ "dep_extreme_2080": {
36
+ "gdb": "dep_extreme_2080.gdb",
37
+ "label": "DEP Extreme Stormwater (3.66 in/hr, 2080 SLR)",
38
+ },
39
+ "dep_moderate_2050": {
40
+ "gdb": "dep_moderate_2050.gdb",
41
+ "label": "DEP Moderate Stormwater (2.13 in/hr, 2050 SLR)",
42
+ },
43
+ "dep_moderate_current": {
44
+ "gdb": "dep_moderate_current.gdb",
45
+ "label": "DEP Moderate Stormwater (2.13 in/hr, current SLR)",
46
+ },
47
+ }
48
+
49
+ DEPTH_CLASS = {
50
+ 1: "Nuisance (>4 in to 1 ft)",
51
+ 2: "Deep & Contiguous (1-4 ft)",
52
+ 3: "Deep Contiguous (>4 ft)",
53
+ }
54
+
55
+
56
+ @lru_cache(maxsize=4)
57
+ def load(scenario: str) -> gpd.GeoDataFrame:
58
+ s = SCENARIOS[scenario]
59
+ path = ROOT / s["gdb"]
60
+ g = gpd.read_file(str(path))
61
+ if g.crs.to_string() != NYC_CRS:
62
+ g = g.to_crs(NYC_CRS)
63
+ return g
64
+
65
+
66
+ def join(assets: gpd.GeoDataFrame, scenario: str) -> gpd.GeoDataFrame:
67
+ """Per-asset depth class, or 0 if outside scenario.
68
+
69
+ Returns a frame indexed like assets with columns: depth_class, depth_label.
70
+ Higher class wins on overlap.
71
+ """
72
+ z = load(scenario)
73
+ a = assets[["geometry"]].copy()
74
+ a["_aid"] = range(len(a))
75
+ j = gpd.sjoin(a, z[["Flooding_Category", "geometry"]],
76
+ how="left", predicate="intersects")
77
+ # for each asset, take max category hit (3 dominates 1)
78
+ cat = (j.groupby("_aid")["Flooding_Category"].max()
79
+ .reindex(range(len(a)))
80
+ .fillna(0).astype(int))
81
+ out = a[["_aid"]].copy()
82
+ out["depth_class"] = cat.values
83
+ out["depth_label"] = out["depth_class"].map(lambda c: DEPTH_CLASS.get(c, "outside"))
84
+ return out[["depth_class", "depth_label"]].reset_index(drop=True)
85
+
86
+
87
+ def label(scenario: str) -> str:
88
+ return SCENARIOS[scenario]["label"]
89
+
90
+
91
+ def _raster_handles():
92
+ """Per-thread rasterio handle cache. rasterio.DatasetReader is not
93
+ safe to share across threads for concurrent .sample() calls; the
94
+ FSM runs each request on its own executor thread, so we keep one
95
+ handle set per thread."""
96
+ h = getattr(_TLOCAL, "handles", None)
97
+ if h is not None:
98
+ return h
99
+ import rasterio
100
+ h = {}
101
+ for s in SCENARIOS:
102
+ p = BAKED / f"{s}.tif"
103
+ if not p.exists():
104
+ return None
105
+ h[s] = rasterio.open(str(p))
106
+ _TLOCAL.handles = h
107
+ return h
108
+
109
+
110
+ def join_raster(pt_geom_2263, scenario: str) -> int:
111
+ """Fast path. Returns the integer depth class (0=outside, 1/2/3) for a
112
+ single shapely Point in EPSG:2263. Falls back to the GDB join() path
113
+ if baked rasters are missing β€” emits a one-time warning so local dev
114
+ still works without the bake artifacts."""
115
+ global _FALLBACK_WARNED
116
+ h = _raster_handles()
117
+ if h is None:
118
+ if not _FALLBACK_WARNED:
119
+ log.warning(
120
+ "data/baked/dep_*.tif not found β€” falling back to GDB sjoin. "
121
+ "Run: uv run python scripts/bake_cornerstone_rasters.py"
122
+ )
123
+ _FALLBACK_WARNED = True
124
+ # legacy fallback β€” wrap point in a one-row GeoDataFrame
125
+ a = gpd.GeoDataFrame(geometry=[pt_geom_2263], crs=NYC_CRS)
126
+ return int(join(a, scenario).iloc[0]["depth_class"])
127
+ ds = h[scenario]
128
+ v = next(ds.sample([(pt_geom_2263.x, pt_geom_2263.y)]))
129
+ return int(v[0])
130
+
131
+
132
+ def coverage_for_polygon(polygon, scenario: str,
133
+ polygon_crs: str = "EPSG:4326") -> dict:
134
+ """Polygon-level summary: what fraction of the input polygon falls into
135
+ each depth class for a given DEP scenario? Used in neighborhood mode.
136
+
137
+ Returns:
138
+ {
139
+ 'scenario': scenario id,
140
+ 'label': human-readable scenario name,
141
+ 'fraction_any': fraction of polygon inside any flooded class,
142
+ 'fraction_class': {1: f, 2: f, 3: f} fraction in each class,
143
+ 'polygon_area_m2': total polygon area,
144
+ }
145
+ """
146
+ z = load(scenario)
147
+ poly_gdf = gpd.GeoDataFrame(geometry=[polygon], crs=polygon_crs).to_crs(NYC_CRS)
148
+ poly_geom = poly_gdf.iloc[0].geometry
149
+ poly_ft2 = float(poly_geom.area)
150
+ sqft_to_m2 = 0.092903
151
+ fraction_class = {1: 0.0, 2: 0.0, 3: 0.0}
152
+ if poly_ft2:
153
+ for cat in (1, 2, 3):
154
+ sub = z[z["Flooding_Category"] == cat]
155
+ if sub.empty:
156
+ continue
157
+ inter = sub.geometry.intersection(poly_geom)
158
+ inter = inter[~inter.is_empty]
159
+ ft2 = float(inter.area.sum()) if len(inter) else 0.0
160
+ fraction_class[cat] = round(ft2 / poly_ft2, 4)
161
+ fraction_any = round(sum(fraction_class.values()), 4)
162
+ return {
163
+ "scenario": scenario,
164
+ "label": label(scenario),
165
+ "fraction_any": fraction_any,
166
+ "fraction_class": fraction_class,
167
+ "polygon_area_m2": round(poly_ft2 * sqft_to_m2, 1),
168
+ }
app/flood_layers/ida_hwm.py ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Hurricane Ida (Sept 2021) empirical flood extent β€” USGS high-water marks.
2
+
3
+ This specialist plays the same role as Prithvi-EO 2.0 (Sen1Floods11)
4
+ in the parent triangulation-engine: it provides empirical post-event
5
+ flood evidence (versus the modeled scenarios from FEMA/DEP). Where
6
+ Prithvi derives extent from Sentinel-1 SAR, USGS HWMs are surveyed
7
+ ground-truth water marks. Both are valid empirical signals; HWMs
8
+ are the public record for Ida specifically.
9
+
10
+ Output per address: number of HWMs within radius, max water elevation
11
+ (ft), nearest site description.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import json
16
+ import math
17
+ from dataclasses import dataclass
18
+ from functools import lru_cache
19
+ from pathlib import Path
20
+
21
+ DATA = Path(__file__).resolve().parent.parent.parent / "data" / "ida_2021_hwms_ny.geojson"
22
+ DOC_ID = "ida_hwm"
23
+ CITATION = "USGS STN Hurricane Ida 2021 high-water marks (Event 312, NY)"
24
+
25
+
26
+ @dataclass
27
+ class HWMSummary:
28
+ n_within_radius: int
29
+ radius_m: int
30
+ max_elev_ft: float | None
31
+ max_height_above_gnd_ft: float | None
32
+ nearest_dist_m: float | None
33
+ nearest_site: str | None
34
+ nearest_elev_ft: float | None
35
+ sample_sites: list[str]
36
+ points: list[dict] | None = None # per-mark for the map layer
37
+
38
+
39
+ def _haversine_m(lat1, lon1, lat2, lon2):
40
+ R = 6371000.0
41
+ p1, p2 = math.radians(lat1), math.radians(lat2)
42
+ dp = math.radians(lat2 - lat1); dl = math.radians(lon2 - lon1)
43
+ a = math.sin(dp / 2) ** 2 + math.cos(p1) * math.cos(p2) * math.sin(dl / 2) ** 2
44
+ return 2 * R * math.asin(math.sqrt(a))
45
+
46
+
47
+ @lru_cache(maxsize=1)
48
+ def _load() -> list[dict]:
49
+ if not DATA.exists():
50
+ return []
51
+ with open(DATA) as f:
52
+ return json.load(f).get("features", [])
53
+
54
+
55
+ def summary_for_point(lat: float, lon: float, radius_m: int = 1000) -> HWMSummary | None:
56
+ feats = _load()
57
+ if not feats:
58
+ return None
59
+ in_radius = []
60
+ nearest = (None, float("inf"), None)
61
+ for f in feats:
62
+ flon, flat = f["geometry"]["coordinates"]
63
+ d = _haversine_m(lat, lon, flat, flon)
64
+ if d <= radius_m:
65
+ in_radius.append((d, f))
66
+ if d < nearest[1]:
67
+ nearest = (f, d, None)
68
+ nf, nd, _ = nearest
69
+ elevs = [f["properties"].get("elev_ft") for _, f in in_radius
70
+ if f["properties"].get("elev_ft") is not None]
71
+ heights = [f["properties"].get("height_above_gnd") for _, f in in_radius
72
+ if f["properties"].get("height_above_gnd") is not None]
73
+ sites = [f["properties"].get("site_description") for _, f in in_radius]
74
+ sites = [s for s in sites if s][:5]
75
+ points = []
76
+ for d, f in in_radius[:50]: # cap so SSE payload stays small
77
+ flon, flat = f["geometry"]["coordinates"]
78
+ p = f["properties"]
79
+ points.append({
80
+ "lat": flat, "lon": flon,
81
+ "site": p.get("site_description"),
82
+ "elev_ft": p.get("elev_ft"),
83
+ "height_above_gnd_ft": p.get("height_above_gnd"),
84
+ "distance_m": round(d, 1),
85
+ })
86
+ return HWMSummary(
87
+ n_within_radius=len(in_radius),
88
+ radius_m=radius_m,
89
+ max_elev_ft=round(max(elevs), 2) if elevs else None,
90
+ max_height_above_gnd_ft=round(max(heights), 2) if heights else None,
91
+ nearest_dist_m=round(nd, 0) if nf is not None else None,
92
+ nearest_site=nf["properties"].get("site_description") if nf else None,
93
+ nearest_elev_ft=nf["properties"].get("elev_ft") if nf else None,
94
+ sample_sites=sites,
95
+ points=points,
96
+ )
app/flood_layers/prithvi_live.py ADDED
@@ -0,0 +1,563 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Prithvi-EO 2.0 (NYC Pluvial v2 fine-tune) live water segmentation.
2
+
3
+ A per-query specialist: pulls the most recent low-cloud Sentinel-2 L2A
4
+ scene over the address from Microsoft Planetary Computer, runs the
5
+ NYC-specialized fine-tune, and reports % water within 500 m.
6
+
7
+ Distinct from `app/flood_layers/prithvi_water.py`, which serves the
8
+ offline-precomputed 2021 Ida polygons. This one is *fresh observation*
9
+ each query β€” same doc_id (`prithvi_live`), but the underlying model
10
+ has been swapped from the Sen1Floods11 base to
11
+ `msradam/Prithvi-EO-2.0-NYC-Pluvial` (Apache-2.0, fine-tuned on AMD
12
+ Instinct MI300X via AMD Developer Cloud β€” test flood IoU 0.5979,
13
+ 6Γ— over the base). The base model is still loadable by setting
14
+ RIPRAP_PRITHVI_LIVE_REPO to the IBM repo as a fallback.
15
+
16
+ Network calls (STAC search + COG band reads) and a 300M-param model
17
+ forward pass make this the slowest specialist after the LLM. Gated by
18
+ RIPRAP_PRITHVI_LIVE_ENABLE so deployments without the deps installed
19
+ silently skip it. Cloud-cover refuses out at 30%+ to honor the
20
+ Sen1Floods11 training distribution.
21
+
22
+ License: Apache-2.0. See experiments/shared/licenses.md.
23
+ """
24
+
25
+ from __future__ import annotations
26
+
27
+ import concurrent.futures
28
+ import logging
29
+ import os
30
+ import threading
31
+ import time
32
+ from typing import Any
33
+
34
+ log = logging.getLogger("riprap.prithvi_live")
35
+
36
+ ENABLE = os.environ.get("RIPRAP_PRITHVI_LIVE_ENABLE", "1").lower() in ("1", "true", "yes")
37
+ SEARCH_DAYS = int(os.environ.get("RIPRAP_PRITHVI_LIVE_SEARCH_DAYS", "120"))
38
+ MAX_CLOUD_PCT = float(os.environ.get("RIPRAP_PRITHVI_LIVE_MAX_CLOUD", "30"))
39
+ DEVICE = os.environ.get("RIPRAP_PRITHVI_LIVE_DEVICE", "cpu")
40
+
41
+ # Default to the NYC Pluvial v2 fine-tune; override to the IBM-NASA base
42
+ # (`ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11`) when the v2
43
+ # artifact is unreachable or for A/B comparisons.
44
+ REPO = os.environ.get(
45
+ "RIPRAP_PRITHVI_LIVE_REPO",
46
+ "msradam/Prithvi-EO-2.0-NYC-Pluvial",
47
+ )
48
+ BASE_REPO = "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11"
49
+
50
+ # Sen1Floods11 expects 6 bands in this exact order.
51
+ BANDS = ["B02", "B03", "B04", "B8A", "B11", "B12"]
52
+ IMG_SIZE = 512 # Sen1Floods11 training crop
53
+ CHIP_PX = 1024
54
+ CHIP_M = CHIP_PX * 10
55
+ HALF_M = CHIP_M / 2
56
+ CENTER_RADIUS_M = 500
57
+ PIXEL_M = 10
58
+
59
+ _MODEL = None
60
+ _RUN_MODEL = None
61
+ _INIT_LOCK = threading.Lock() # serializes lazy load if multiple threads
62
+ # hit fetch() before _MODEL is populated
63
+
64
+
65
+ def _has_required_deps() -> tuple[bool, str | None]:
66
+ """Probe deps in two tiers.
67
+
68
+ Tier 1 β€” chip fetching (planetary_computer / pystac_client / rioxarray
69
+ / xarray / einops) is always required: prithvi_live always pulls a
70
+ Sentinel-2 chip from Microsoft Planetary Computer regardless of where
71
+ inference runs.
72
+
73
+ Tier 2 β€” local inference (terratorch) is only required when remote
74
+ inference is unavailable. On the HF Space we have remote inference
75
+ on the AMD MI300X via app/inference.py, so terratorch is not needed
76
+ even though chip-fetch is.
77
+
78
+ Returns (False, missing) if any required dep is missing. Splitting
79
+ the gate this way lets the HF Space deployment fetch chips and run
80
+ remote inference even though it doesn't fit terratorch's transitive
81
+ dep cone (~250 MB) in the HF build sandbox."""
82
+ chip_deps = ("planetary_computer", "pystac_client",
83
+ "rioxarray", "xarray", "einops")
84
+ missing = [n for n in chip_deps
85
+ if not _has_module(n)]
86
+ if missing:
87
+ return False, ", ".join(missing)
88
+ # Tier 2: only need terratorch if we'd run inference locally.
89
+ try:
90
+ from app import inference as _inf
91
+ if _inf.remote_enabled():
92
+ return True, None
93
+ except Exception:
94
+ pass
95
+ if not _has_module("terratorch"):
96
+ return False, "terratorch (local inference)"
97
+ return True, None
98
+
99
+
100
+ def _has_module(name: str) -> bool:
101
+ """True if `name` imports cleanly. ImportError β†’ not installed.
102
+ Other exceptions (e.g. torchvision::nms RuntimeError on the HF
103
+ Space) β†’ treat as unavailable too; we don't want a clean-skip
104
+ intent to crash the FSM at deps-probe time."""
105
+ try:
106
+ __import__(name)
107
+ return True
108
+ except ImportError:
109
+ return False
110
+ except Exception as e:
111
+ log.warning("prithvi_live: %s import raised %s; treating as "
112
+ "unavailable", name, type(e).__name__)
113
+ return False
114
+
115
+
116
+ _DEPS_OK, _DEPS_MISSING = _has_required_deps()
117
+
118
+
119
+ def warm():
120
+ """Optional pre-load. The FSM action is lazy too β€” calling warm()
121
+ here just amortizes the first-query cost at app boot."""
122
+ if not ENABLE:
123
+ return
124
+ try:
125
+ _ensure_model()
126
+ except Exception:
127
+ log.exception("prithvi_live: warm() failed; specialist will no-op")
128
+
129
+
130
+ def _ensure_model():
131
+ """Load Prithvi-EO 2.0 once into RAM.
132
+
133
+ The v2 NYC Pluvial fine-tune (`msradam/Prithvi-EO-2.0-NYC-Pluvial`)
134
+ is **architecturally distinct** from the IBM-NASA Sen1Floods11
135
+ base: v2 ships a `UNetDecoder` + 2-class head, the base ships a
136
+ UperNet with PSP / FPN. The model has to be built from each
137
+ repo's own config.yaml β€” there's no key-mapping shim that bridges
138
+ them.
139
+
140
+ Strategy:
141
+
142
+ 1. If the active REPO != BASE_REPO, try to build from the v2
143
+ yaml + v2 ckpt. The v2 yaml's data: paths point at the
144
+ training droplet's filesystem (`/root/terramind_nyc/...`)
145
+ which doesn't exist locally; that's fine β€” the
146
+ GenericNonGeoSegmentationDataModule constructor only
147
+ records the paths, splits aren't read until `setup()`.
148
+ 2. On any v2 failure (yaml not present, datamodule constructor
149
+ strict, weights mismatch), fall back to the base yaml + base
150
+ ckpt. The base path is the proven pre-C5 behaviour.
151
+
152
+ The shared `inference.run_model` helper is only published by the
153
+ IBM-NASA base repo; we always pull it from there.
154
+ """
155
+ global _MODEL, _RUN_MODEL
156
+ if _MODEL is not None:
157
+ return _MODEL, _RUN_MODEL
158
+ with _INIT_LOCK:
159
+ if _MODEL is not None: # double-check inside the lock
160
+ return _MODEL, _RUN_MODEL
161
+ import importlib.util
162
+
163
+ from huggingface_hub import hf_hub_download
164
+ from terratorch.cli_tools import LightningInferenceModel
165
+ log.info("prithvi_live: loading model from %s", REPO)
166
+
167
+ # Inference helper only lives in the IBM-NASA base repo.
168
+ inference_py = hf_hub_download(BASE_REPO, "inference.py")
169
+
170
+ m = None
171
+ # ---- v2 path: yaml + ckpt from the published repo ----------
172
+ if REPO != BASE_REPO:
173
+ try:
174
+ # The v2 repo publishes `prithvi_nyc_phase14.yaml` and
175
+ # `prithvi_nyc_pluvial_v2.ckpt`. Be tolerant of small
176
+ # naming drift (best_val_loss.ckpt etc.) by probing.
177
+ v2_yaml = None
178
+ for name in ("prithvi_nyc_phase14.yaml",
179
+ "config.yaml", "phase14.yaml",
180
+ "prithvi_nyc_v2.yaml"):
181
+ try:
182
+ v2_yaml = hf_hub_download(REPO, name)
183
+ break
184
+ except Exception:
185
+ continue
186
+ v2_ckpt = None
187
+ for name in ("prithvi_nyc_pluvial_v2.ckpt",
188
+ "best_val_loss.ckpt", "model.ckpt",
189
+ "last.ckpt"):
190
+ try:
191
+ v2_ckpt = hf_hub_download(REPO, name)
192
+ break
193
+ except Exception:
194
+ continue
195
+ if v2_yaml and v2_ckpt:
196
+ log.info("prithvi_live: building v2 model from "
197
+ "yaml=%s ckpt=%s", v2_yaml, v2_ckpt)
198
+ m = LightningInferenceModel.from_config(v2_yaml, v2_ckpt)
199
+ # prithvi_nyc_phase14.yaml uses GenericNonGeoSegmentationDataModule
200
+ # which omits test_transform (β†’ None) and uses terratorch Normalize
201
+ # for aug (only handles 4D/5D). IBM inference.py:run_model() calls
202
+ # both on a 3D dict. Patch both to match the IBM base contract:
203
+ # ToTensorV2 for test_transform; Kornia AugmentationSequential
204
+ # (accepts dict input, adds batch dim) for aug.
205
+ if getattr(getattr(m, 'datamodule', None),
206
+ 'test_transform', None) is None:
207
+ import albumentations as A
208
+ import torch as _torch
209
+ from albumentations.pytorch import ToTensorV2
210
+ m.datamodule.test_transform = A.Compose([ToTensorV2()])
211
+ _old = m.datamodule.aug
212
+
213
+ # IBM's inference.py:188 calls
214
+ # `datamodule.aug({'image': tensor})['image']`.
215
+ # kornia's AugmentationSequential doesn't accept
216
+ # dict input cleanly and tripped the
217
+ # `'list' object has no attribute 'view'`
218
+ # error on the L4 deploy. Use a hand-rolled
219
+ # dict-aware normalizer instead β€” same math,
220
+ # fewer moving parts, no kornia version skew.
221
+ class _DictNormalize:
222
+ def __init__(self, mean, std):
223
+ self.mean = _torch.as_tensor(mean).view(-1, 1, 1).float()
224
+ self.std = _torch.as_tensor(std).view(-1, 1, 1).float()
225
+
226
+ def __call__(self, sample):
227
+ if isinstance(sample, dict):
228
+ img = sample["image"]
229
+ mean = self.mean.to(img.device)
230
+ std = self.std.to(img.device)
231
+ return {**sample, "image": (img - mean) / std}
232
+ mean = self.mean.to(sample.device)
233
+ std = self.std.to(sample.device)
234
+ return (sample - mean) / std
235
+
236
+ # `_old.means` / `_old.stds` come from the
237
+ # yaml as Python lists β€” calling `.view()` on
238
+ # them is what tripped the original
239
+ # `'list' object has no attribute 'view'`.
240
+ # _DictNormalize handles the conversion via
241
+ # torch.as_tensor internally; just pass the
242
+ # raw values whatever their type.
243
+ m.datamodule.aug = _DictNormalize(_old.means, _old.stds)
244
+ log.info("prithvi_live: patched v2 datamodule transforms "
245
+ "for IBM inference.py compat (dict-aware Normalize)")
246
+ else:
247
+ log.warning("prithvi_live: v2 yaml/ckpt not "
248
+ "discoverable in %s; falling back to base",
249
+ REPO)
250
+ except Exception as e:
251
+ log.warning("prithvi_live: v2 build failed (%s); "
252
+ "falling back to base", e)
253
+ m = None
254
+
255
+ # ---- base path: proven IBM-NASA Sen1Floods11 fine-tune -----
256
+ if m is None:
257
+ base_config = hf_hub_download(BASE_REPO, "config.yaml")
258
+ base_ckpt = hf_hub_download(
259
+ BASE_REPO, "Prithvi-EO-V2-300M-TL-Sen1Floods11.pt")
260
+ m = LightningInferenceModel.from_config(base_config, base_ckpt)
261
+
262
+ m.model.eval()
263
+ if DEVICE == "cuda":
264
+ try:
265
+ import torch
266
+ if torch.cuda.is_available():
267
+ m.model.cuda()
268
+ except Exception:
269
+ log.exception("prithvi_live: cuda move failed")
270
+
271
+ spec = importlib.util.spec_from_file_location("_prithvi_inference",
272
+ inference_py)
273
+ mod = importlib.util.module_from_spec(spec)
274
+ spec.loader.exec_module(mod)
275
+ _MODEL = m
276
+ _RUN_MODEL = mod.run_model
277
+ return _MODEL, _RUN_MODEL
278
+
279
+
280
+ def _search_recent_scene(lat: float, lon: float):
281
+ """Most recent low-cloud S2 L2A item near (lat, lon) in the last
282
+ SEARCH_DAYS days, or None."""
283
+ import datetime as dt
284
+
285
+ import planetary_computer as pc
286
+ from pystac_client import Client
287
+ end = dt.datetime.utcnow().date()
288
+ start = end - dt.timedelta(days=SEARCH_DAYS)
289
+ client = Client.open(
290
+ "https://planetarycomputer.microsoft.com/api/stac/v1",
291
+ modifier=pc.sign_inplace,
292
+ )
293
+ delta = 0.02
294
+ search = client.search(
295
+ collections=["sentinel-2-l2a"],
296
+ bbox=[lon - delta, lat - delta, lon + delta, lat + delta],
297
+ datetime=f"{start}/{end}",
298
+ query={"eo:cloud_cover": {"lt": MAX_CLOUD_PCT}},
299
+ max_items=20,
300
+ )
301
+ items = sorted(
302
+ search.items(),
303
+ key=lambda it: (it.properties.get("eo:cloud_cover", 100),
304
+ -(it.datetime.timestamp() if it.datetime else 0)),
305
+ )
306
+ return items[0] if items else None
307
+
308
+
309
+ def _build_chip(item, lat: float, lon: float):
310
+ """Returns (img, ref_da, epsg) β€” img is the (6, H, W) center-cropped
311
+ float32 array; ref_da is the rioxarray DataArray of the reference
312
+ band BEFORE the center crop (kept so we can compute the affine
313
+ transform for polygonization in EPSG:4326)."""
314
+ import numpy as np
315
+ import rioxarray # noqa: F401
316
+ import xarray as xr
317
+ from pyproj import Transformer
318
+ if "proj:epsg" in item.properties:
319
+ epsg = int(item.properties["proj:epsg"])
320
+ else:
321
+ code = item.properties.get("proj:code", "")
322
+ if code.startswith("EPSG:"):
323
+ epsg = int(code.split(":", 1)[1])
324
+ else:
325
+ raise RuntimeError("STAC item missing proj:epsg / proj:code")
326
+ fwd = Transformer.from_crs("EPSG:4326", f"EPSG:{epsg}", always_xy=True)
327
+ cx, cy = fwd.transform(lon, lat)
328
+ xmin, xmax = cx - HALF_M, cx + HALF_M
329
+ ymin, ymax = cy - HALF_M, cy + HALF_M
330
+ ref = rioxarray.open_rasterio(item.assets[BANDS[0]].href, masked=False).squeeze(drop=True)
331
+ ref = ref.rio.clip_box(minx=xmin, miny=ymin, maxx=xmax, maxy=ymax)
332
+ ref = ref.isel(y=slice(0, CHIP_PX), x=slice(0, CHIP_PX))
333
+ arrs = [ref.astype("float32")]
334
+ for b in BANDS[1:]:
335
+ da = rioxarray.open_rasterio(item.assets[b].href, masked=False).squeeze(drop=True)
336
+ da = da.rio.clip_box(minx=xmin, miny=ymin, maxx=xmax, maxy=ymax)
337
+ if da.shape != ref.shape:
338
+ da = da.rio.reproject_match(ref)
339
+ arrs.append(da.astype("float32"))
340
+ stacked = xr.concat(arrs, dim="band", join="override").assign_coords(band=BANDS)
341
+ img = stacked.values # (6, H, W)
342
+ # Center crop to IMG_SIZE x IMG_SIZE.
343
+ _, h, w = img.shape
344
+ sy, sx = (h - IMG_SIZE) // 2, (w - IMG_SIZE) // 2
345
+ img = img[:, sy:sy + IMG_SIZE, sx:sx + IMG_SIZE]
346
+ if img.mean() > 1:
347
+ img = img / 10000.0
348
+ return np.nan_to_num(img.astype("float32")), ref, epsg
349
+
350
+
351
+ def _polygonize_mask(pred, ref_da, epsg: int) -> dict | None:
352
+ """Vectorize the binary water mask into an EPSG:4326 GeoJSON
353
+ FeatureCollection so the frontend can paint it on the MapLibre
354
+ map. Returns None on failure (best-effort β€” never raises into the
355
+ caller path)."""
356
+ try:
357
+ import json
358
+
359
+ import geopandas as gpd
360
+ from rasterio.features import shapes
361
+ from rasterio.transform import from_origin
362
+ from shapely.geometry import shape
363
+ # Reconstruct the affine transform of the center-cropped pred.
364
+ # ref_da has 1024 px at 10 m; we cropped to the central 512.
365
+ xs = ref_da.x.values
366
+ ys = ref_da.y.values
367
+ if len(xs) < IMG_SIZE or len(ys) < IMG_SIZE:
368
+ return None
369
+ # rioxarray gives pixel-centered coords; offset by half a pixel
370
+ # to the upper-left to build a from_origin transform.
371
+ sy = (len(ys) - IMG_SIZE) // 2
372
+ sx = (len(xs) - IMG_SIZE) // 2
373
+ # ys are descending (top-to-bottom); take the top of the crop.
374
+ top_y = float(ys[sy]) + (PIXEL_M / 2.0)
375
+ left_x = float(xs[sx]) - (PIXEL_M / 2.0)
376
+ transform = from_origin(left_x, top_y, PIXEL_M, PIXEL_M)
377
+ # Polygonize only the water class (1).
378
+ mask = (pred == 1).astype("uint8")
379
+ polys = []
380
+ for geom, value in shapes(mask, mask=mask.astype(bool),
381
+ transform=transform):
382
+ if value != 1:
383
+ continue
384
+ polys.append(shape(geom))
385
+ if not polys:
386
+ return {"type": "FeatureCollection", "features": []}
387
+ gdf = gpd.GeoDataFrame({"geometry": polys},
388
+ crs=f"EPSG:{epsg}").to_crs("EPSG:4326")
389
+ # Simplify slightly to keep the SSE payload small (10 m raster
390
+ # over 5 km square = up to ~10 k tiny squares; simplification
391
+ # collapses adjacent water pixels into smooth polygons).
392
+ gdf["geometry"] = gdf.geometry.simplify(0.00005, preserve_topology=True)
393
+ return json.loads(gdf.to_json())
394
+ except Exception:
395
+ log.exception("prithvi_live: polygonize failed")
396
+ return None
397
+
398
+
399
+ def _fetch_inner(lat: float, lon: float, timeout_s: float) -> dict[str, Any]:
400
+ """Core fetch logic β€” run inside a bounded thread via fetch()."""
401
+ t0 = time.time()
402
+ try:
403
+ item = _search_recent_scene(lat, lon)
404
+ if item is None:
405
+ return {"ok": False, "skipped": f"no <{MAX_CLOUD_PCT}% cloud "
406
+ f"S2 in last {SEARCH_DAYS}d"}
407
+ cc = float(item.properties.get("eo:cloud_cover", -1))
408
+ if time.time() - t0 > timeout_s:
409
+ return {"ok": False, "skipped": "stac search exceeded budget"}
410
+ img, ref_da, epsg = _build_chip(item, lat, lon)
411
+ if time.time() - t0 > timeout_s:
412
+ return {"ok": False, "skipped": "chip build exceeded budget"}
413
+
414
+ # v0.4.5 β€” try the MI300X inference service first if configured.
415
+ # On RemoteUnreachable (service down / not configured / 5xx) fall
416
+ # through to the local terratorch path. When remote is configured
417
+ # but returns non-ok we surface that signal directly: the local
418
+ # path on this machine has been brittle (v2 datamodule
419
+ # `test_transform=None` race), so a configured remote is more
420
+ # reliable than the fallback.
421
+ remote_attempted = False
422
+ try:
423
+ from app import inference as _inf
424
+ if _inf.remote_enabled():
425
+ remote_attempted = True
426
+ remote = _inf.prithvi_pluvial(
427
+ img, scene_id=item.id,
428
+ scene_datetime=str(item.datetime),
429
+ cloud_cover=cc,
430
+ timeout=timeout_s,
431
+ )
432
+ if remote.get("ok"):
433
+ # Vectorize the remote prediction raster so the map
434
+ # actually renders the live water polygons. The
435
+ # droplet returns `pred_b64` (uint8 binary mask);
436
+ # we polygonize against the chip's WGS84 bounds
437
+ # which we know locally from `ref_da`.
438
+ polys = None
439
+ pred_b64 = remote.get("pred_b64")
440
+ pred_shape = remote.get("pred_shape")
441
+ if pred_b64 and pred_shape:
442
+ try:
443
+ xs = ref_da.x.values
444
+ ys = ref_da.y.values
445
+ from pyproj import Transformer
446
+ t_inv = Transformer.from_crs(
447
+ f"EPSG:{epsg}", "EPSG:4326",
448
+ always_xy=True)
449
+ minx, maxx = float(xs.min()), float(xs.max())
450
+ miny, maxy = float(ys.min()), float(ys.max())
451
+ minlon, minlat = t_inv.transform(minx, miny)
452
+ maxlon, maxlat = t_inv.transform(maxx, maxy)
453
+ from app.context._polygonize import (
454
+ polygonize_binary_mask,
455
+ )
456
+ polys = polygonize_binary_mask(
457
+ pred_b64, pred_shape,
458
+ (minlon, minlat, maxlon, maxlat),
459
+ label="water", fill_color="#1F77B4",
460
+ simplify_tolerance=2e-5,
461
+ )
462
+ except Exception:
463
+ log.exception("prithvi_live: remote polygonize failed")
464
+ polys = None
465
+ return {
466
+ "ok": True,
467
+ "item_id": item.id,
468
+ "item_datetime": str(item.datetime),
469
+ "cloud_cover": cc,
470
+ "pct_water_full": remote.get("pct_water_full"),
471
+ "pct_water_within_500m": remote.get("pct_water_within_500m"),
472
+ "polygons_geojson": polys,
473
+ "compute": f"remote Β· {remote.get('device', 'gpu')}",
474
+ "elapsed_s": round(time.time() - t0, 2),
475
+ }
476
+ err = (remote.get("err")
477
+ or remote.get("error")
478
+ or remote.get("skipped")
479
+ or "unknown")
480
+ return {"ok": False,
481
+ "skipped": f"remote prithvi-pluvial non-ok: {err}",
482
+ "elapsed_s": round(time.time() - t0, 2)}
483
+ except _inf.RemoteUnreachable as e:
484
+ log.info("prithvi_live: remote unreachable (%s)", e)
485
+ if remote_attempted:
486
+ # Don't fall to local β€” torchvision::nms is broken on the
487
+ # CPU-tier UI Spaces and crashes the FSM specialist with
488
+ # a confusing RuntimeError. Return a clean skipped row so
489
+ # the trace says "remote unreachable" instead.
490
+ return {"ok": False,
491
+ "skipped": f"remote prithvi-pluvial unreachable: {e}",
492
+ "elapsed_s": round(time.time() - t0, 2)}
493
+ except Exception as e:
494
+ log.exception("prithvi_live: remote call failed")
495
+ if remote_attempted:
496
+ return {"ok": False,
497
+ "skipped": f"remote prithvi-pluvial error: "
498
+ f"{type(e).__name__}: {e}",
499
+ "elapsed_s": round(time.time() - t0, 2)}
500
+
501
+ # Local fallback β€” the path that's been live since v0.4.4.
502
+ # Reached only when remote_attempted is False (i.e. remote
503
+ # backend not configured at all).
504
+ model, run_model = _ensure_model()
505
+ x = img[None, :, None, :, :] # (1, 6, 1, H, W)
506
+ pred_t = run_model(x, None, None, model.model, model.datamodule, IMG_SIZE)
507
+ import numpy as np
508
+ pred = pred_t[0].cpu().numpy().astype("uint8")
509
+ pct_full = float(100.0 * pred.mean())
510
+ yy, xx = np.indices(pred.shape)
511
+ cy, cx = pred.shape[0] // 2, pred.shape[1] // 2
512
+ radius_px = CENTER_RADIUS_M / PIXEL_M
513
+ circle = (yy - cy) ** 2 + (xx - cx) ** 2 <= radius_px ** 2
514
+ pct_500 = float(100.0 * pred[circle].mean()) if circle.sum() else 0.0
515
+ polygons_geojson = _polygonize_mask(pred, ref_da, epsg)
516
+ return {
517
+ "ok": True,
518
+ "item_id": item.id,
519
+ "item_datetime": str(item.datetime),
520
+ "cloud_cover": cc,
521
+ "pct_water_full": pct_full,
522
+ "pct_water_within_500m": pct_500,
523
+ "polygons_geojson": polygons_geojson,
524
+ "compute": "local",
525
+ "elapsed_s": round(time.time() - t0, 2),
526
+ }
527
+ except Exception as e:
528
+ log.exception("prithvi_live: fetch failed")
529
+ return {"ok": False, "err": f"{type(e).__name__}: {e}",
530
+ "elapsed_s": round(time.time() - t0, 2)}
531
+
532
+
533
+ def fetch(lat: float, lon: float, timeout_s: float = 60.0) -> dict[str, Any]:
534
+ """Run the specialist. Wraps _fetch_inner in a bounded thread so that
535
+ STAC searches and COG band reads (which lack per-request HTTP timeouts)
536
+ cannot hang the FSM indefinitely.
537
+
538
+ Returns a dict with at minimum:
539
+ { "ok": bool, "skipped": str | None, "item_id": str | None,
540
+ "cloud_cover": float | None, "pct_water_within_500m": float | None }
541
+ Designed to never raise; failures show up as ok=False with an `err`.
542
+ """
543
+ if not ENABLE:
544
+ return {"ok": False, "skipped": "RIPRAP_PRITHVI_LIVE_ENABLE=0"}
545
+ if not _DEPS_OK:
546
+ return {"ok": False,
547
+ "skipped": f"deps unavailable on this deployment: "
548
+ f"{_DEPS_MISSING}"}
549
+ hard_timeout = timeout_s + 15.0
550
+ from app import emissions as _emissions
551
+ _parent_tracker = _emissions.current()
552
+ with concurrent.futures.ThreadPoolExecutor(
553
+ max_workers=1,
554
+ initializer=lambda t=_parent_tracker: _emissions.install(t),
555
+ ) as pool:
556
+ future = pool.submit(_fetch_inner, lat, lon, timeout_s)
557
+ try:
558
+ return future.result(timeout=hard_timeout)
559
+ except concurrent.futures.TimeoutError:
560
+ log.warning("prithvi_live: hard timeout after %.0fs (STAC/COG hung)",
561
+ hard_timeout)
562
+ return {"ok": False,
563
+ "skipped": f"prithvi_live timed out after {hard_timeout:.0f}s"}
app/flood_layers/prithvi_water.py ADDED
@@ -0,0 +1,120 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Prithvi-EO 2.0 (Sen1Floods11) satellite flood inundation specialist.
2
+
3
+ The 300M-parameter Prithvi-EO foundation model (NASA/IBM, Apache-2.0)
4
+ was run twice offline on Hurricane Ida 2021 pre/post HLS Sentinel-2
5
+ scenes over central NYC:
6
+
7
+ pre : HLS.S30.T18TWK.2021237T153809 (2021-08-25, 3% cloud)
8
+ post: HLS.S30.T18TWK.2021245T154911 (2021-09-02, 1% cloud,
9
+ ~12 hours after peak rainfall)
10
+
11
+ The diff (post-water minus pre-water, filtered to β‰₯3-cell polygons)
12
+ isolates surface water present 12 hours after Ida that wasn't present
13
+ the prior week β€” i.e., candidate Ida-attributable inundation. We ship
14
+ the resulting polygons as a flood-layer specialist; per query we
15
+ compute proximity from the address to the nearest such polygon.
16
+
17
+ Honest scope:
18
+ - Sub-surface flooding (subway entrances, basement apartments β€” the
19
+ dominant Ida damage mode in NYC) is not visible to optical satellites.
20
+ - Pluvial street water had largely drained by the Sep 2 16:02Z pass,
21
+ so the residual Prithvi signal mostly captures marsh ponding,
22
+ riverside spillover, and low-lying park inundation.
23
+ - The model fired on Ida itself (a real flood event), not a synthetic
24
+ fallback β€” that's the architectural value.
25
+ """
26
+ from __future__ import annotations
27
+
28
+ import json
29
+ import math
30
+ from dataclasses import dataclass
31
+ from functools import lru_cache
32
+ from pathlib import Path
33
+
34
+ DATA_DIR = Path(__file__).resolve().parent.parent.parent / "data"
35
+ DOC_ID = "prithvi_water"
36
+ CITATION = ("Prithvi-EO-2.0-300M-TL-Sen1Floods11 (NASA/IBM, Apache-2.0, via "
37
+ "TerraTorch). Hurricane Ida pre/post diff: pre HLS T18TWK "
38
+ "2021-08-25 (3% cloud), post HLS T18TWK 2021-09-02 (1% cloud, "
39
+ "~12h after peak rainfall).")
40
+
41
+
42
+ @dataclass
43
+ class PrithviSummary:
44
+ inside_water_polygon: bool
45
+ nearest_distance_m: float | None
46
+ n_polygons_within_500m: int
47
+ scene_id: str
48
+ scene_date: str
49
+
50
+
51
+ def _haversine_m(lat1, lon1, lat2, lon2):
52
+ R = 6371000.0
53
+ p1, p2 = math.radians(lat1), math.radians(lat2)
54
+ dp = math.radians(lat2 - lat1); dl = math.radians(lon2 - lon1)
55
+ a = math.sin(dp / 2) ** 2 + math.cos(p1) * math.cos(p2) * math.sin(dl / 2) ** 2
56
+ return 2 * R * math.asin(math.sqrt(a))
57
+
58
+
59
+ @lru_cache(maxsize=1)
60
+ def _load():
61
+ """Load the merged Prithvi water mask (combined across NYC MGRS tiles)
62
+ as a GeoDataFrame in NYC state plane (EPSG:2263) for fast metric
63
+ distance queries."""
64
+ import geopandas as gpd
65
+ # Prefer the Ida flood-event diff (real flood-attribution signal);
66
+ # fall back to clear-day permanent-water masks if the Ida file is absent.
67
+ candidates = [
68
+ DATA_DIR / "prithvi_ida_2021.geojson",
69
+ DATA_DIR / "prithvi_flood_nyc.geojson",
70
+ ]
71
+ candidates += sorted(DATA_DIR.glob("prithvi_flood_*.geojson"), reverse=True)
72
+ path = next((p for p in candidates if p.exists()), None)
73
+ if path is None:
74
+ return None, None
75
+ with open(path) as f:
76
+ meta = json.load(f)
77
+ g = gpd.read_file(path)
78
+ if g.crs is None:
79
+ g.set_crs("EPSG:4326", inplace=True)
80
+ g = g.to_crs("EPSG:2263")
81
+ return g, meta
82
+
83
+
84
+ def warm() -> None:
85
+ _load()
86
+
87
+
88
+ def summary_for_point(lat: float, lon: float) -> PrithviSummary | None:
89
+ import geopandas as gpd
90
+ from shapely.geometry import Point
91
+ g, meta = _load()
92
+ if g is None:
93
+ return None
94
+ pt_wgs = gpd.GeoSeries([Point(lon, lat)], crs="EPSG:4326")
95
+ pt_2263 = pt_wgs.to_crs("EPSG:2263").iloc[0]
96
+ inside = bool(g.contains(pt_2263).any())
97
+
98
+ # nearest distance (feet -> metres)
99
+ distances_ft = g.geometry.distance(pt_2263)
100
+ nearest_ft = float(distances_ft.min()) if len(distances_ft) else None
101
+ nearest_m = round(nearest_ft / 3.281, 1) if nearest_ft is not None else None
102
+
103
+ within_500m = int((distances_ft <= 500 * 3.281).sum())
104
+
105
+ # The Ida pre/post artifact carries pre_/post_ scene info; the clear-day
106
+ # artifact carries scene_ids[]. Format compactly for either case.
107
+ if "post_scene_id" in meta:
108
+ sid = f"pre {meta['pre_scene_id']} | post {meta['post_scene_id']}"
109
+ sdate = f"pre {meta['pre_scene_date']}, post {meta['post_scene_date']}"
110
+ else:
111
+ sid = meta.get("scene_id") or ", ".join(meta.get("scene_ids", []) or ["unknown"])
112
+ sdate = meta.get("scene_date") or ", ".join(meta.get("scene_dates", []) or ["unknown"])
113
+
114
+ return PrithviSummary(
115
+ inside_water_polygon=inside,
116
+ nearest_distance_m=nearest_m,
117
+ n_polygons_within_500m=within_500m,
118
+ scene_id=sid,
119
+ scene_date=sdate,
120
+ )
app/flood_layers/sandy_inundation.py ADDED
@@ -0,0 +1,110 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """NYC Sandy Inundation Zone (empirical 2012 extent, NYC OD 5xsi-dfpx).
2
+
3
+ Two query paths exist:
4
+ inside_raster(point) β€” fast path. Samples data/baked/sandy.tif.
5
+ ~1 ms; used by step_sandy in the FSM.
6
+ join(assets) β€” legacy GeoJSON sjoin path. Retained as a
7
+ fallback when the baked raster is absent (local dev) and
8
+ for coverage_for_polygon (neighborhood mode).
9
+ """
10
+ from __future__ import annotations
11
+
12
+ import logging
13
+ import threading
14
+ from functools import lru_cache
15
+
16
+ import geopandas as gpd
17
+
18
+ from app.spatial import DATA, NYC_CRS, load_layer
19
+
20
+ DOC_ID = "sandy_inundation"
21
+ CITATION = "NYC Sandy Inundation Zone (NYC OpenData 5xsi-dfpx, empirical 2012 extent)"
22
+
23
+ log = logging.getLogger(__name__)
24
+ BAKED = DATA / "baked"
25
+ _TLOCAL = threading.local()
26
+ _FALLBACK_WARNED = False
27
+
28
+
29
+ @lru_cache(maxsize=1)
30
+ def load() -> gpd.GeoDataFrame:
31
+ g = load_layer(DATA / "sandy_inundation.geojson")
32
+ return g[["geometry"]]
33
+
34
+
35
+ def join(assets: gpd.GeoDataFrame) -> gpd.pd.Series:
36
+ """Return a boolean Series indexed like assets: True if inside Sandy zone."""
37
+ z = load()
38
+ # spatial join avoids fragile unary union over messy public polygons
39
+ hits = gpd.sjoin(
40
+ assets[["geometry"]].assign(_aid=range(len(assets))),
41
+ z[["geometry"]],
42
+ how="left",
43
+ predicate="intersects",
44
+ )
45
+ flagged = hits.dropna(subset=["index_right"])["_aid"].unique()
46
+ s = assets.geometry.copy().astype(bool)
47
+ s[:] = False
48
+ s.iloc[list(flagged)] = True
49
+ return s.reset_index(drop=True)
50
+
51
+
52
+ def _raster_handle():
53
+ """Per-thread rasterio handle. See dep_stormwater._raster_handles."""
54
+ h = getattr(_TLOCAL, "handle", None)
55
+ if h is not None:
56
+ return h
57
+ p = BAKED / "sandy.tif"
58
+ if not p.exists():
59
+ return None
60
+ import rasterio
61
+ h = rasterio.open(str(p))
62
+ _TLOCAL.handle = h
63
+ return h
64
+
65
+
66
+ def inside_raster(pt_geom_2263) -> bool:
67
+ """Fast path. True if the shapely Point (in EPSG:2263) falls inside the
68
+ 2012 Sandy inundation extent. Falls back to the GeoJSON sjoin path if
69
+ data/baked/sandy.tif is missing."""
70
+ global _FALLBACK_WARNED
71
+ h = _raster_handle()
72
+ if h is None:
73
+ if not _FALLBACK_WARNED:
74
+ log.warning(
75
+ "data/baked/sandy.tif not found β€” falling back to GeoJSON sjoin. "
76
+ "Run: uv run python scripts/bake_cornerstone_rasters.py"
77
+ )
78
+ _FALLBACK_WARNED = True
79
+ a = gpd.GeoDataFrame(geometry=[pt_geom_2263], crs=NYC_CRS)
80
+ return bool(join(a).iloc[0])
81
+ v = next(h.sample([(pt_geom_2263.x, pt_geom_2263.y)]))
82
+ return bool(int(v[0]))
83
+
84
+
85
+ def coverage_for_polygon(polygon, polygon_crs: str = "EPSG:4326") -> dict:
86
+ """Polygon-level summary: what fraction of the input polygon overlaps
87
+ the 2012 Sandy inundation extent? Used in neighborhood-mode queries.
88
+
89
+ Returns:
90
+ {
91
+ 'overlap_area_m2': absolute overlap in m2,
92
+ 'polygon_area_m2': total polygon area in m2,
93
+ 'fraction': overlap / polygon_area, range [0, 1],
94
+ 'inside': True if any overlap exists,
95
+ }
96
+ """
97
+ z = load().to_crs("EPSG:2263") # NY State Plane Long Island, units = ft
98
+ poly_gdf = gpd.GeoDataFrame(geometry=[polygon], crs=polygon_crs).to_crs("EPSG:2263")
99
+ poly_geom = poly_gdf.iloc[0].geometry
100
+ inter = z.intersection(poly_geom)
101
+ inter = inter[~inter.is_empty]
102
+ overlap_ft2 = float(inter.area.sum()) if len(inter) else 0.0
103
+ poly_ft2 = float(poly_geom.area)
104
+ sqft_to_m2 = 0.092903
105
+ return {
106
+ "overlap_area_m2": round(overlap_ft2 * sqft_to_m2, 1),
107
+ "polygon_area_m2": round(poly_ft2 * sqft_to_m2, 1),
108
+ "fraction": round(overlap_ft2 / poly_ft2, 4) if poly_ft2 else 0.0,
109
+ "inside": overlap_ft2 > 0,
110
+ }
app/framing.py ADDED
@@ -0,0 +1,249 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Question-aware framing for the Capstone briefing opening.
2
+
3
+ The four-section structure (Status / Empirical / Modeled / Policy) is
4
+ load-bearing for the Mellea grounding checks and stays unchanged. What
5
+ this module does is detect the *shape* of the user's question from the
6
+ raw query string + planner intent, then return a single-sentence
7
+ directive that conditions only the opening Status sentence.
8
+
9
+ Eleven question types are recognised; they mirror the rubric in
10
+ `tests/integration/stakeholder_queries.py:FRAMING_RUBRICS`. Detection
11
+ is deterministic regex matching β€” no extra LLM call, no added latency.
12
+
13
+ Usage:
14
+
15
+ from app.framing import augment_system_prompt
16
+ system_prompt = augment_system_prompt(
17
+ EXTRA_SYSTEM_PROMPT, query=user_query, intent=plan.intent,
18
+ )
19
+
20
+ The returned prompt has the original text plus a trailing
21
+ `QUESTION-AWARE OPENING:` block. Granite 4.1 attends to this through
22
+ the system-prompt cache and applies it to the Status sentence.
23
+ """
24
+ from __future__ import annotations
25
+
26
+ import re
27
+ from typing import Final
28
+
29
+ QUESTION_TYPES: Final[tuple[str, ...]] = (
30
+ "habitability_decision",
31
+ "legal_disclosure",
32
+ "capital_planning",
33
+ "underwriting",
34
+ "journalism",
35
+ "development_siting",
36
+ "grant_evidence",
37
+ "retrospective",
38
+ "emergency_response",
39
+ "comparison",
40
+ "generic_exposure",
41
+ )
42
+
43
+
44
+ # ---- Per-type opening directives ------------------------------------------
45
+ #
46
+ # Each directive is one sentence that supplements (does not replace) the
47
+ # Status section's existing instruction. Granite 4.1 has a strong prior
48
+ # toward "this address is exposed to ..." openings; the directive
49
+ # overrides that in a question-shaped way without disturbing the four
50
+ # grounding invariants.
51
+
52
+ _DIRECTIVES: dict[str, str] = {
53
+ "habitability_decision": (
54
+ "The Status sentence MUST start with a direct verdict word "
55
+ "(\"Yes\" if the documents show meaningful flood evidence, \"No\" "
56
+ "if they don't), then name the single strongest piece of "
57
+ "evidence with its [doc_id]. The user is deciding whether to "
58
+ "live here β€” answer the question, then cite."
59
+ ),
60
+ "legal_disclosure": (
61
+ "The Status sentence MUST state whether the documents contain "
62
+ "facts a NY RPL Β§462(2) or Β§231-b disclosure would need to "
63
+ "record. Begin with \"Disclosure is warranted\" or \"Disclosure "
64
+ "is not triggered\" based on the evidence, then name the "
65
+ "specific fact with its [doc_id]. The user is a real-estate "
66
+ "professional checking the disclosure threshold."
67
+ ),
68
+ "capital_planning": (
69
+ "The Status sentence MUST frame the place as a capital-planning "
70
+ "candidate: name the dominant exposure with its [doc_id] and "
71
+ "indicate whether the evidence supports prioritization "
72
+ "(\"merits prioritization\", \"ranks high for hardening\") or "
73
+ "not. The user allocates infrastructure investment."
74
+ ),
75
+ "underwriting": (
76
+ "The Status sentence MUST emphasize that every figure in the "
77
+ "briefing is independently sourced β€” open with the dominant "
78
+ "exposure and the specific [doc_id], then add a half-clause "
79
+ "noting that the audit chain follows below. The user is an "
80
+ "underwriter who needs a defensible loss narrative."
81
+ ),
82
+ "journalism": (
83
+ "The Status sentence MUST be reproducible reporting prose: "
84
+ "name the place, name the dominant exposure with [doc_id], "
85
+ "and avoid editorial verbs like \"shocking\" or \"alarming\". "
86
+ "The user is a data journalist who will cite this prose verbatim."
87
+ ),
88
+ "development_siting": (
89
+ "The Status sentence MUST start with the count of active "
90
+ "construction filings cited from [dob_permits] (e.g. \"N "
91
+ "active construction filings sit inside ...\") and indicate "
92
+ "which flood layer they intersect. The user is a developer or "
93
+ "architect doing a pre-design siting check."
94
+ ),
95
+ "grant_evidence": (
96
+ "The Status sentence MUST open with \"Vulnerability "
97
+ "assessment:\" and name the place + dominant exposure with "
98
+ "[doc_id]. Treat the briefing as the evidence section of a "
99
+ "HUD CDBG-DR or FEMA BRIC application β€” formal, third-person, "
100
+ "free of advocacy framing."
101
+ ),
102
+ "retrospective": (
103
+ "Riprap currently runs on present-day data sources. The Status "
104
+ "sentence MUST acknowledge the question is retrospective and "
105
+ "state explicitly that the briefing reflects the CURRENT state "
106
+ "of these data sources, not a snapshot from the requested date. "
107
+ "Then proceed with the present-day exposure picture so the user "
108
+ "still gets the geography. Silence-over-confabulation: never "
109
+ "reconstruct historical conditions you can't verify."
110
+ ),
111
+ "emergency_response": (
112
+ "The Status sentence MUST quantify what is at risk in the "
113
+ "next few hours, citing the live signal that triggered the "
114
+ "query and any active alerts with [doc_id]. The user needs an "
115
+ "operational picture, not a historical exposure summary."
116
+ ),
117
+ "comparison": (
118
+ "The Status sentence MUST name BOTH places the user is "
119
+ "comparing and indicate which one shows greater exposure on "
120
+ "the strongest cited signal. If only one place's data is "
121
+ "available in the documents, say so explicitly. The user is "
122
+ "doing a head-to-head decision."
123
+ ),
124
+ "generic_exposure": "", # default β€” no override
125
+ }
126
+
127
+
128
+ # ---- Detector -------------------------------------------------------------
129
+ #
130
+ # Patterns are ordered: the FIRST type whose pattern matches wins. Order
131
+ # matters β€” more specific question shapes (legal_disclosure, grant_evidence,
132
+ # emergency_response) come before more general ones (habitability_decision,
133
+ # capital_planning) so the obvious specialist tags don't get swallowed.
134
+
135
+ _PATTERNS: list[tuple[str, list[re.Pattern]]] = [
136
+ ("retrospective", [
137
+ re.compile(r"\b(would have|would Riprap|on (the )?date of|as of (the )?(date|day)|"
138
+ r"day before|prior to|before (Hurricane|Ida|Sandy|the storm)|"
139
+ r"on (August|September|October|November|December|January|February|March|"
140
+ r"April|May|June|July) \d{1,2},? ?\d{4}|"
141
+ r"time.?machine|retrospective|court (exhibit|testimony))\b", re.I),
142
+ ]),
143
+ ("emergency_response", [
144
+ re.compile(r"\b(just triggered|right now|next (few |six |\d+ )?hours?|"
145
+ r"in the next \d+|currently flooding|flood (warning|watch) is active|"
146
+ r"sensor [A-Z]{2}-?\d+|live (alert|trigger))\b", re.I),
147
+ ]),
148
+ ("legal_disclosure", [
149
+ re.compile(r"\b(disclos(e|ure|ed)|RPL\s*Β§?\s*\d+|Property Condition Disclosure|"
150
+ r"Β§\s*462|Β§\s*231-?b|seller'?s? disclosure|landlord'?s? disclosure|"
151
+ r"required to disclose|need to disclose)\b", re.I),
152
+ ]),
153
+ ("grant_evidence", [
154
+ re.compile(r"\b(vulnerability assessment|CDBG-?DR|HUD|BRIC|"
155
+ r"grant application|funding application|community resilience grant|"
156
+ r"FEMA application|disaster recovery (application|funding))\b", re.I),
157
+ ]),
158
+ ("development_siting", [
159
+ re.compile(r"\b(what (are|is) (they|being) build(ing)?|new construction|"
160
+ r"under construction|active (construction|filing|project|permit)|"
161
+ r"projects? (in progress|underway|planned)|architects?|"
162
+ r"siting check|pre.?design|"
163
+ r"DOB filing|developer)\b", re.I),
164
+ ]),
165
+ ("comparison", [
166
+ # `prioritize X over Y` can have many words between, hence the
167
+ # bounded non-greedy span β€” capped at 80 chars to avoid runaway.
168
+ re.compile(r"\b(compare\b|comparison|\bvs\b|\bversus\b|"
169
+ r"head-?to-?head|\brank\s+the\s+top)\b", re.I),
170
+ re.compile(r"\bprioritize\b.{1,80}\bover\b", re.I | re.S),
171
+ re.compile(r"\bover\s+\w+(?:\s+\w+){0,3}\s+for\s+(hardening|investment)\b", re.I),
172
+ ]),
173
+ ("capital_planning", [
174
+ re.compile(r"\b(prioritiz(e|ation)|capital plan(ning)?|harden(ing|s)?|"
175
+ r"infrastructure investment|where (should|to) (we |the )(invest|"
176
+ r"prioritize|harden)|MTA.+prioritize|DEP.+prioritize|"
177
+ r"protection envelope|outside (it|the protection)|"
178
+ r"resilien(ce|cy) project)\b", re.I),
179
+ ]),
180
+ ("habitability_decision", [
181
+ re.compile(r"\b(should I worry|should I (be|consider)|is (it|this) safe|"
182
+ r"can I (rent|live|move|raise (my )?kids?)|considering (renting|leasing|moving)|"
183
+ r"(thinking about|planning to) (rent|lease|move|buy)|"
184
+ r"is (this|that|the landlord) true|landlord (says|claims|told)|"
185
+ r"no flood history|just got a lease|new lease|signing a lease|"
186
+ r"\bworry\b)", re.I),
187
+ ]),
188
+ ("underwriting", [
189
+ re.compile(r"\b(underwrit(e|er|ing|able)|actuarial|loss history|"
190
+ r"insurabl[ey]|catastrophe (model|risk)|"
191
+ r"insurance (audit|memo|profile)|"
192
+ r"audit (chain|trail))\b", re.I),
193
+ ]),
194
+ ("journalism", [
195
+ re.compile(r"\b(reporter|journalist|newsroom|story|coverage|"
196
+ r"published?|publish (this|the))", re.I),
197
+ ]),
198
+ ]
199
+
200
+
201
+ def detect(query: str, intent: str | None = None) -> str:
202
+ """Classify the question shape from the raw query and planner intent.
203
+
204
+ Returns one of `QUESTION_TYPES`. Falls back to `generic_exposure`
205
+ when no pattern matches β€” that's the existing behavior, preserved.
206
+
207
+ `intent` is currently advisory only (the patterns don't read it),
208
+ but the parameter is part of the API so future refinements can
209
+ use it (e.g. an `intent=neighborhood` query without a verdict
210
+ keyword could default to `journalism` rather than `generic_exposure`).
211
+ """
212
+ if not query:
213
+ return "generic_exposure"
214
+ q = query.strip()
215
+ for qt, patterns in _PATTERNS:
216
+ if any(p.search(q) for p in patterns):
217
+ return qt
218
+ # Heuristic fallback: bare neighborhood/borough names from a planner
219
+ # context default to journalism (most common stakeholder reading a
220
+ # neighborhood-only query is a reporter or planner). For
221
+ # single_address with no question keyword, fall back to generic.
222
+ if intent == "neighborhood" and len(q.split()) <= 3:
223
+ return "journalism"
224
+ return "generic_exposure"
225
+
226
+
227
+ def opening_instruction(question_type: str) -> str:
228
+ """Return the directive sentence(s) for a question type.
229
+ Returns empty string for `generic_exposure` (no override)."""
230
+ return _DIRECTIVES.get(question_type, "")
231
+
232
+
233
+ def augment_system_prompt(base: str, *, query: str,
234
+ intent: str | None = None) -> str:
235
+ """Wrap a base system prompt with a question-aware opening directive.
236
+
237
+ No-op when the detector returns `generic_exposure` β€” the original
238
+ behavior is preserved.
239
+ """
240
+ qt = detect(query, intent)
241
+ directive = opening_instruction(qt)
242
+ if not directive:
243
+ return base
244
+ return (
245
+ f"{base}\n\n"
246
+ f"QUESTION-AWARE OPENING (this directive overrides ONLY the opening "
247
+ f"**Status.** sentence; the four-section structure and citation "
248
+ f"discipline above remain in force):\n{directive}"
249
+ )
app/fsm.py ADDED
@@ -0,0 +1,1394 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Riprap Burr FSM β€” linear specialist pipeline for one address.
2
+
3
+ Each action either produces a structured fact (which becomes a document
4
+ the reconciler can cite) or stays silent on failure. The reconciler
5
+ (Granite 4.1) only sees documents from specialists that actually
6
+ produced data β€” the silence-over-confabulation contract.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import logging
11
+ import threading as _threading
12
+ import time
13
+ from typing import Any
14
+
15
+ import geopandas as gpd
16
+ from burr.core import ApplicationBuilder, State, action
17
+ from shapely.geometry import Point
18
+
19
+ from app import emissions
20
+ from app.context import floodnet, microtopo, noaa_tides, npcc4_slr, nws_alerts, nws_obs, nyc311
21
+ from app.energy import estimate as energy_estimate
22
+ from app.flood_layers import dep_stormwater, ida_hwm, prithvi_water, sandy_inundation
23
+ from app.geocode import geocode_one
24
+ from app.live import floodnet_forecast as fn_forecast
25
+ from app.live import ttm_forecast
26
+ from app.rag import retrieve as rag_retrieve
27
+ from app.reconcile import citations_from_docs, reconcile as run_reconcile
28
+ from app.registers import doe_schools as r_schools
29
+ from app.registers import doh_hospitals as r_hospitals
30
+ from app.registers import mta_entrances as r_mta
31
+ from app.registers import nycha as r_nycha
32
+
33
+ log = logging.getLogger("riprap.fsm")
34
+
35
+ # NYC five-borough bbox. Specialists whose data sources are NYC-only
36
+ # (Sandy 2012, NYC DEP Stormwater, FloodNet, NYC 311, NYC microtopo
37
+ # raster, NYC Hurricane Ida Prithvi polygons) skip with an explicit
38
+ # "out of NYC scope" reason when geocode lands outside this envelope.
39
+ # Live specialists (NWS / NOAA / TTM) and the NY-State Ida HWMs run
40
+ # unconditionally.
41
+ _NYC_S, _NYC_W, _NYC_N, _NYC_E = 40.49, -74.27, 40.92, -73.69
42
+
43
+
44
+ def _in_nyc(lat, lon) -> bool:
45
+ if lat is None or lon is None:
46
+ return False
47
+ return _NYC_S <= lat <= _NYC_N and _NYC_W <= lon <= _NYC_E
48
+
49
+ # Thread-local hook so the streaming endpoint can subscribe to per-token
50
+ # Granite output during reconcile, without threading a callback through
51
+ # every Burr action signature.
52
+ _FSM_LOCAL = _threading.local()
53
+
54
+
55
+ def set_token_callback(on_token):
56
+ """Install a per-thread on_token(delta) callable for the next reconcile.
57
+ Pass None to clear."""
58
+ _FSM_LOCAL.on_token = on_token
59
+
60
+
61
+ def _current_token_callback():
62
+ return getattr(_FSM_LOCAL, "on_token", None)
63
+
64
+
65
+ def set_mellea_attempt_callback(fn):
66
+ _FSM_LOCAL.on_mellea_attempt = fn
67
+
68
+
69
+ def _current_mellea_attempt_callback():
70
+ return getattr(_FSM_LOCAL, "on_mellea_attempt", None)
71
+
72
+
73
+ def set_strict_mode(strict: bool):
74
+ """Per-thread flag β€” when True the linear FSM's reconcile step routes
75
+ through Mellea-validated rejection sampling instead of the standard
76
+ streaming reconciler. Disables token streaming for that step."""
77
+ _FSM_LOCAL.strict = bool(strict)
78
+
79
+
80
+ def _current_strict_mode() -> bool:
81
+ return bool(getattr(_FSM_LOCAL, "strict", False))
82
+
83
+
84
+ def set_planned_specialists(spec_names):
85
+ """Install a per-thread set of specialist names from the planner.
86
+
87
+ Used by step_reconcile to trim doc messages: documents whose family
88
+ prefix doesn't match any planned specialist are dropped before the
89
+ Mellea call. Cuts ~30-50% of prompt tokens on local Ollama, where
90
+ the FSM otherwise hands the reconciler every specialist's output
91
+ even if the planner only asked for a subset."""
92
+ _FSM_LOCAL.planned_specialists = set(spec_names) if spec_names else None
93
+
94
+
95
+ def _current_planned_specialists():
96
+ return getattr(_FSM_LOCAL, "planned_specialists", None)
97
+
98
+
99
+ def set_user_query(query: str | None):
100
+ """Install the user's original natural-language query for question-aware
101
+ framing in step_reconcile. The FSM's state["query"] is the geocoder
102
+ input (often just the street address), which doesn't carry the
103
+ user's question shape β€” set this separately so Capstone can detect
104
+ 'should I worry' / 'is disclosure required' / etc."""
105
+ _FSM_LOCAL.user_query = query
106
+
107
+
108
+ def _current_user_query() -> str | None:
109
+ return getattr(_FSM_LOCAL, "user_query", None)
110
+
111
+
112
+ def set_planner_intent(intent: str | None):
113
+ """Install the planner's classified intent so step_reconcile can pass
114
+ it to the framing detector as a tiebreaker on bare-place queries."""
115
+ _FSM_LOCAL.planner_intent = intent
116
+
117
+
118
+ def _current_planner_intent() -> str | None:
119
+ return getattr(_FSM_LOCAL, "planner_intent", None)
120
+
121
+
122
+ # Canonical Burr: one action per specialist, sequential transitions.
123
+ # A previous version of this module wrapped 16 specialists in a single
124
+ # fan-out action that ran them concurrently in a ThreadPoolExecutor;
125
+ # that path was removed because it sometimes hung after the fan-out
126
+ # completed (Burr-internal post-action cleanup with custom executors)
127
+ # and made the trace UI's per-step timing harder to reason about.
128
+ # Parallelism, when wanted, belongs at the inference layer
129
+ # (vLLM / Ollama NUM_PARALLEL), not the FSM.
130
+
131
+ def _step(state: State, name: str) -> dict[str, Any]:
132
+ """Append a step record to the trace; returns the dict so the action
133
+ can mutate timing/result fields."""
134
+ trace = list(state.get("trace", []))
135
+ rec = {"step": name, "started_at": time.time(), "ok": None}
136
+ trace.append(rec)
137
+ return rec, trace
138
+
139
+
140
+ @action(reads=["query"], writes=["geocode", "lat", "lon", "trace"])
141
+ def step_geocode(state: State) -> State:
142
+ rec, trace = _step(state, "geocode")
143
+ try:
144
+ hit = geocode_one(state["query"])
145
+ if hit is None:
146
+ rec["ok"] = False
147
+ rec["err"] = "no geocoder match"
148
+ # Burr requires every declared write to be populated. Emit
149
+ # explicit None rather than leaving keys absent.
150
+ return state.update(geocode=None, lat=None, lon=None, trace=trace)
151
+ rec["ok"] = True
152
+ rec["result"] = {"address": hit.address, "lat": hit.lat, "lon": hit.lon}
153
+ return state.update(
154
+ geocode={"address": hit.address, "borough": hit.borough,
155
+ "lat": hit.lat, "lon": hit.lon,
156
+ "bbl": hit.bbl, "bin": hit.bin},
157
+ lat=hit.lat, lon=hit.lon, trace=trace,
158
+ )
159
+ except Exception as e:
160
+ rec["ok"] = False
161
+ rec["err"] = str(e)
162
+ log.exception("geocode failed")
163
+ return state.update(geocode=None, lat=None, lon=None, trace=trace)
164
+ finally:
165
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
166
+
167
+
168
+ @action(reads=["lat", "lon"], writes=["sandy", "trace"])
169
+ def step_sandy(state: State) -> State:
170
+ rec, trace = _step(state, "sandy_inundation")
171
+ try:
172
+ if state.get("lat") is None:
173
+ rec["ok"] = False; rec["err"] = "no coords"
174
+ return state.update(sandy=None, trace=trace)
175
+ if not _in_nyc(state["lat"], state["lon"]):
176
+ rec["ok"] = False; rec["err"] = "out of NYC scope"
177
+ return state.update(sandy=None, trace=trace)
178
+ pt_geom = (gpd.GeoDataFrame(geometry=[Point(state["lon"], state["lat"])],
179
+ crs="EPSG:4326")
180
+ .to_crs("EPSG:2263").iloc[0].geometry)
181
+ flag = sandy_inundation.inside_raster(pt_geom)
182
+ rec["ok"] = True; rec["result"] = {"inside": flag}
183
+ return state.update(sandy=flag, trace=trace)
184
+ except Exception as e:
185
+ rec["ok"] = False; rec["err"] = str(e)
186
+ log.exception("sandy failed")
187
+ return state.update(sandy=None, trace=trace)
188
+ finally:
189
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
190
+
191
+
192
+ @action(reads=["lat", "lon"], writes=["dep", "trace"])
193
+ def step_dep(state: State) -> State:
194
+ rec, trace = _step(state, "dep_stormwater")
195
+ try:
196
+ if state.get("lat") is None:
197
+ rec["ok"] = False; rec["err"] = "no coords"
198
+ return state.update(dep=None, trace=trace)
199
+ if not _in_nyc(state["lat"], state["lon"]):
200
+ rec["ok"] = False; rec["err"] = "out of NYC scope"
201
+ return state.update(dep=None, trace=trace)
202
+ pt_geom = (gpd.GeoDataFrame(geometry=[Point(state["lon"], state["lat"])],
203
+ crs="EPSG:4326")
204
+ .to_crs("EPSG:2263").iloc[0].geometry)
205
+ out: dict[str, Any] = {}
206
+ for scen in ["dep_extreme_2080", "dep_moderate_2050", "dep_moderate_current"]:
207
+ cls = dep_stormwater.join_raster(pt_geom, scen)
208
+ out[scen] = {
209
+ "depth_class": cls,
210
+ "depth_label": dep_stormwater.DEPTH_CLASS.get(cls, "outside"),
211
+ "citation": f"NYC DEP Stormwater Flood Map β€” {dep_stormwater.label(scen)}",
212
+ }
213
+ rec["ok"] = True; rec["result"] = {k: v["depth_label"] for k, v in out.items()}
214
+ return state.update(dep=out, trace=trace)
215
+ except Exception as e:
216
+ rec["ok"] = False; rec["err"] = str(e)
217
+ log.exception("dep failed")
218
+ return state.update(dep=None, trace=trace)
219
+ finally:
220
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
221
+
222
+
223
+ @action(reads=["lat", "lon"], writes=["floodnet", "trace"])
224
+ def step_floodnet(state: State) -> State:
225
+ rec, trace = _step(state, "floodnet")
226
+ try:
227
+ if state.get("lat") is None:
228
+ rec["ok"] = False; rec["err"] = "no coords"
229
+ return state.update(floodnet=None, trace=trace)
230
+ if not _in_nyc(state["lat"], state["lon"]):
231
+ rec["ok"] = False; rec["err"] = "out of NYC scope"
232
+ return state.update(floodnet=None, trace=trace)
233
+ s = floodnet.summary_for_point(state["lat"], state["lon"], radius_m=600)
234
+ s["radius_m"] = 600
235
+ rec["ok"] = True
236
+ rec["result"] = {"n_sensors": s["n_sensors"],
237
+ "n_events_3y": s["n_flood_events_3y"]}
238
+ return state.update(floodnet=s, trace=trace)
239
+ except Exception as e:
240
+ rec["ok"] = False; rec["err"] = str(e)
241
+ log.exception("floodnet failed")
242
+ return state.update(floodnet=None, trace=trace)
243
+ finally:
244
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
245
+
246
+
247
+ @action(reads=["lat", "lon"], writes=["nyc311", "trace"])
248
+ def step_311(state: State) -> State:
249
+ rec, trace = _step(state, "nyc311")
250
+ try:
251
+ if state.get("lat") is None:
252
+ rec["ok"] = False; rec["err"] = "no coords"
253
+ return state.update(nyc311=None, trace=trace)
254
+ if not _in_nyc(state["lat"], state["lon"]):
255
+ rec["ok"] = False; rec["err"] = "out of NYC scope"
256
+ return state.update(nyc311=None, trace=trace)
257
+ s = nyc311.summary_for_point(state["lat"], state["lon"], radius_m=200, years=5)
258
+ rec["ok"] = True; rec["result"] = {"n": s["n"]}
259
+ return state.update(nyc311=s, trace=trace)
260
+ except Exception as e:
261
+ rec["ok"] = False; rec["err"] = str(e)
262
+ log.exception("311 failed")
263
+ return state.update(nyc311=None, trace=trace)
264
+ finally:
265
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
266
+
267
+
268
+ @action(reads=["lat", "lon"], writes=["ida_hwm", "trace"])
269
+ def step_ida_hwm(state: State) -> State:
270
+ rec, trace = _step(state, "ida_hwm_2021")
271
+ try:
272
+ if state.get("lat") is None:
273
+ rec["ok"] = False; rec["err"] = "no coords"
274
+ return state.update(ida_hwm=None, trace=trace)
275
+ s = ida_hwm.summary_for_point(state["lat"], state["lon"], radius_m=800)
276
+ if s is None:
277
+ rec["ok"] = False; rec["err"] = "HWM data missing"
278
+ return state.update(ida_hwm=None, trace=trace)
279
+ rec["ok"] = True
280
+ rec["result"] = {
281
+ "n_within_800m": s.n_within_radius,
282
+ "max_height_above_gnd_ft": s.max_height_above_gnd_ft,
283
+ "nearest_m": s.nearest_dist_m,
284
+ }
285
+ return state.update(ida_hwm=vars(s), trace=trace)
286
+ except Exception as e:
287
+ rec["ok"] = False; rec["err"] = str(e)
288
+ log.exception("ida_hwm failed")
289
+ return state.update(ida_hwm=None, trace=trace)
290
+ finally:
291
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
292
+
293
+
294
+ @action(reads=["lat", "lon"], writes=["prithvi_water", "trace"])
295
+ def step_prithvi(state: State) -> State:
296
+ rec, trace = _step(state, "prithvi_eo_v2")
297
+ try:
298
+ if state.get("lat") is None:
299
+ rec["ok"] = False; rec["err"] = "no coords"
300
+ return state.update(prithvi_water=None, trace=trace)
301
+ if not _in_nyc(state["lat"], state["lon"]):
302
+ rec["ok"] = False; rec["err"] = "out of NYC scope"
303
+ return state.update(prithvi_water=None, trace=trace)
304
+ s = prithvi_water.summary_for_point(state["lat"], state["lon"])
305
+ if s is None:
306
+ rec["ok"] = False; rec["err"] = "Prithvi mask missing"
307
+ return state.update(prithvi_water=None, trace=trace)
308
+ rec["ok"] = True
309
+ rec["result"] = {
310
+ "inside_water_polygon": s.inside_water_polygon,
311
+ "nearest_distance_m": s.nearest_distance_m,
312
+ "n_polygons_within_500m": s.n_polygons_within_500m,
313
+ }
314
+ return state.update(prithvi_water=vars(s), trace=trace)
315
+ except Exception as e:
316
+ rec["ok"] = False; rec["err"] = str(e)
317
+ log.exception("prithvi failed")
318
+ return state.update(prithvi_water=None, trace=trace)
319
+ finally:
320
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
321
+
322
+
323
+ @action(reads=["lat", "lon"], writes=["prithvi_live", "trace"])
324
+ def step_prithvi_live(state: State) -> State:
325
+ """Live Sentinel-2 water segmentation via Prithvi-EO 2.0.
326
+
327
+ Network + 300M-param forward pass per query, so it's the slowest
328
+ specialist by far. Gracefully no-ops via the underlying module if
329
+ `RIPRAP_PRITHVI_LIVE_ENABLE=0` or if STAC / model load fails.
330
+ """
331
+ rec, trace = _step(state, "prithvi_eo_live")
332
+ try:
333
+ if state.get("lat") is None:
334
+ rec["ok"] = False; rec["err"] = "no coords"
335
+ return state.update(prithvi_live=None, trace=trace)
336
+ if not _in_nyc(state["lat"], state["lon"]):
337
+ rec["ok"] = False; rec["err"] = "out of NYC scope"
338
+ return state.update(prithvi_live=None, trace=trace)
339
+ from app.flood_layers import prithvi_live
340
+ s = prithvi_live.fetch(state["lat"], state["lon"])
341
+ rec["ok"] = bool(s.get("ok"))
342
+ if not s.get("ok"):
343
+ rec["err"] = s.get("err") or s.get("skipped") or "no observation"
344
+ else:
345
+ rec["result"] = {
346
+ "scene_date": (s.get("item_datetime") or "")[:10],
347
+ "cloud_cover": s.get("cloud_cover"),
348
+ "pct_water_500m": s.get("pct_water_within_500m"),
349
+ "pct_water_5km": s.get("pct_water_full"),
350
+ }
351
+ return state.update(prithvi_live=s, trace=trace)
352
+ except Exception as e:
353
+ rec["ok"] = False; rec["err"] = str(e)
354
+ log.exception("prithvi_live failed")
355
+ return state.update(prithvi_live=None, trace=trace)
356
+ finally:
357
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
358
+
359
+
360
+ @action(reads=["lat", "lon"], writes=["ttm_311_forecast", "trace"])
361
+ def step_ttm_311_forecast(state: State) -> State:
362
+ """TTM r2 zero-shot forecast on weekly 311 flood-complaint counts
363
+ at this specific address (200 m radius). 52 weeks of context β†’
364
+ 4 weeks of forecast. Per-query, per-address, citable."""
365
+ rec, trace = _step(state, "ttm_311_forecast")
366
+ try:
367
+ if state.get("lat") is None:
368
+ rec["ok"] = False; rec["err"] = "no coords"
369
+ return state.update(ttm_311_forecast=None, trace=trace)
370
+ if not _in_nyc(state["lat"], state["lon"]):
371
+ rec["ok"] = False; rec["err"] = "out of NYC scope"
372
+ return state.update(ttm_311_forecast=None, trace=trace)
373
+ s = ttm_forecast.weekly_311_forecast_for_point(state["lat"], state["lon"])
374
+ rec["ok"] = bool(s.get("available"))
375
+ if not rec["ok"]:
376
+ rec["err"] = s.get("reason", "unavailable")
377
+ else:
378
+ rec["result"] = {
379
+ "history_total": s.get("history_total_complaints"),
380
+ "history_recent_mean": s.get("history_recent_3mo_mean"),
381
+ "forecast_mean": s.get("forecast_mean_per_week"),
382
+ "forecast_peak": s.get("forecast_peak_per_week"),
383
+ "accelerating": s.get("accelerating"),
384
+ }
385
+ return state.update(ttm_311_forecast=s, trace=trace)
386
+ except Exception as e:
387
+ rec["ok"] = False; rec["err"] = str(e)
388
+ log.exception("ttm_311_forecast failed")
389
+ return state.update(ttm_311_forecast=None, trace=trace)
390
+ finally:
391
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
392
+
393
+
394
+ @action(reads=["lat", "lon"], writes=["terramind", "trace"])
395
+ def step_terramind(state: State) -> State:
396
+ """TerraMind v1 base β€” DEM β†’ S2L2A synthesis as a per-query
397
+ cognitive-engine node. ~3-7s on M3 CPU. Output is a
398
+ *synthetic-prior* β€” explicitly fourth epistemic class alongside
399
+ empirical / modeled / proxy. Frame the doc body and reconciler
400
+ narration as 'plausible synthesis from terrain context', never
401
+ 'imaged' or 'reconstructed'."""
402
+ rec, trace = _step(state, "terramind_synthesis")
403
+ try:
404
+ if state.get("lat") is None:
405
+ rec["ok"] = False; rec["err"] = "no coords"
406
+ return state.update(terramind=None, trace=trace)
407
+ if not _in_nyc(state["lat"], state["lon"]):
408
+ rec["ok"] = False; rec["err"] = "out of NYC scope"
409
+ return state.update(terramind=None, trace=trace)
410
+ from app.context import terramind_synthesis
411
+ s = terramind_synthesis.fetch(state["lat"], state["lon"])
412
+ rec["ok"] = bool(s.get("ok"))
413
+ if not s.get("ok"):
414
+ rec["err"] = s.get("err") or s.get("skipped") or "terramind unavailable"
415
+ else:
416
+ rec["result"] = {
417
+ "tim_chain": s.get("tim_chain"),
418
+ "diffusion_steps": s.get("diffusion_steps"),
419
+ "dem_mean_m": s.get("dem_mean_m"),
420
+ "synth_chip_shape": s.get("synth_chip_shape"),
421
+ "elapsed_s": s.get("elapsed_s"),
422
+ }
423
+ return state.update(terramind=s, trace=trace)
424
+ except Exception as e:
425
+ rec["ok"] = False; rec["err"] = str(e)
426
+ log.exception("terramind failed")
427
+ return state.update(terramind=None, trace=trace)
428
+ finally:
429
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
430
+
431
+
432
+ @action(reads=["lat", "lon"], writes=["noaa_tides", "trace"])
433
+ def step_noaa_tides(state: State) -> State:
434
+ rec, trace = _step(state, "noaa_tides")
435
+ try:
436
+ if state.get("lat") is None:
437
+ rec["ok"] = False; rec["err"] = "no coords"
438
+ return state.update(noaa_tides=None, trace=trace)
439
+ s = noaa_tides.summary_for_point(state["lat"], state["lon"])
440
+ rec["ok"] = s.get("error") is None
441
+ rec["result"] = {
442
+ "station": s["station_id"],
443
+ "observed_ft_mllw": s["observed_ft_mllw"],
444
+ "residual_ft": s["residual_ft"],
445
+ }
446
+ if s.get("error"): rec["err"] = s["error"]
447
+ return state.update(noaa_tides=s, trace=trace)
448
+ except Exception as e:
449
+ rec["ok"] = False; rec["err"] = str(e)
450
+ log.exception("noaa_tides failed")
451
+ return state.update(noaa_tides=None, trace=trace)
452
+ finally:
453
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
454
+
455
+
456
+ @action(reads=["lat", "lon"], writes=["nws_alerts", "trace"])
457
+ def step_nws_alerts(state: State) -> State:
458
+ rec, trace = _step(state, "nws_alerts")
459
+ try:
460
+ if state.get("lat") is None:
461
+ rec["ok"] = False; rec["err"] = "no coords"
462
+ return state.update(nws_alerts=None, trace=trace)
463
+ s = nws_alerts.summary_for_point(state["lat"], state["lon"])
464
+ rec["ok"] = s.get("error") is None
465
+ rec["result"] = {"n_active": s["n_active"]}
466
+ if s.get("error"): rec["err"] = s["error"]
467
+ return state.update(nws_alerts=s, trace=trace)
468
+ except Exception as e:
469
+ rec["ok"] = False; rec["err"] = str(e)
470
+ log.exception("nws_alerts failed")
471
+ return state.update(nws_alerts=None, trace=trace)
472
+ finally:
473
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
474
+
475
+
476
+ @action(reads=["lat", "lon"], writes=["nws_obs", "trace"])
477
+ def step_nws_obs(state: State) -> State:
478
+ rec, trace = _step(state, "nws_obs")
479
+ try:
480
+ if state.get("lat") is None:
481
+ rec["ok"] = False; rec["err"] = "no coords"
482
+ return state.update(nws_obs=None, trace=trace)
483
+ s = nws_obs.summary_for_point(state["lat"], state["lon"])
484
+ rec["ok"] = s.get("error") is None
485
+ rec["result"] = {
486
+ "station": s["station_id"],
487
+ "p1h_mm": s["precip_last_hour_mm"],
488
+ "p6h_mm": s["precip_last_6h_mm"],
489
+ }
490
+ if s.get("error"): rec["err"] = s["error"]
491
+ return state.update(nws_obs=s, trace=trace)
492
+ except Exception as e:
493
+ rec["ok"] = False; rec["err"] = str(e)
494
+ log.exception("nws_obs failed")
495
+ return state.update(nws_obs=None, trace=trace)
496
+ finally:
497
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
498
+
499
+
500
+ @action(reads=["lat", "lon"], writes=["ttm_forecast", "trace"])
501
+ def step_ttm_forecast(state: State) -> State:
502
+ """Granite TTM r2 zero-shot forecast of the Battery surge residual."""
503
+ rec, trace = _step(state, "ttm_forecast")
504
+ try:
505
+ if state.get("lat") is None:
506
+ rec["ok"] = False; rec["err"] = "no coords"
507
+ return state.update(ttm_forecast=None, trace=trace)
508
+ s = ttm_forecast.summary_for_point(state["lat"], state["lon"])
509
+ if not s.get("available"):
510
+ rec["ok"] = False
511
+ rec["err"] = s.get("reason", "TTM unavailable")
512
+ return state.update(ttm_forecast=None, trace=trace)
513
+ rec["ok"] = True
514
+ rec["result"] = {
515
+ "context": s["context_length"],
516
+ "horizon": s["horizon_steps"],
517
+ "forecast_peak_ft": s["forecast_peak_ft"],
518
+ "forecast_peak_min_ahead": s["forecast_peak_minutes_ahead"],
519
+ "interesting": s["interesting"],
520
+ }
521
+ return state.update(ttm_forecast=s, trace=trace)
522
+ except Exception as e:
523
+ rec["ok"] = False; rec["err"] = str(e)
524
+ log.exception("ttm_forecast failed")
525
+ return state.update(ttm_forecast=None, trace=trace)
526
+ finally:
527
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
528
+
529
+
530
+ @action(reads=["lat", "lon"], writes=["ttm_battery_surge", "trace"])
531
+ def step_ttm_battery_surge(state: State) -> State:
532
+ """Granite TTM r2 fine-tune β€” 96 h hourly Battery surge nowcast.
533
+
534
+ Same TTM r2 backbone family as step_ttm_forecast but a different
535
+ artefact: msradam/Granite-TTM-r2-Battery-Surge, trained on AMD
536
+ MI300X. Hourly cadence vs the zero-shot's 6-min, 4-day vs 9.6 h
537
+ horizon. Both can fire on the same query β€” the reconciler frames
538
+ each as a distinct forecast in the briefing."""
539
+ rec, trace = _step(state, "ttm_battery_surge")
540
+ try:
541
+ if state.get("lat") is None:
542
+ rec["ok"] = False; rec["err"] = "no coords"
543
+ return state.update(ttm_battery_surge=None, trace=trace)
544
+ # Battery gauge is a single point; the forecast applies citywide
545
+ # to NYC harbor entrance, so we don't gate by NYC bbox.
546
+ from app.live import ttm_battery_surge
547
+ s = ttm_battery_surge.fetch()
548
+ rec["ok"] = bool(s.get("available"))
549
+ if not rec["ok"]:
550
+ rec["err"] = s.get("reason", "unavailable")
551
+ return state.update(ttm_battery_surge=None, trace=trace)
552
+ rec["result"] = {
553
+ "context_h": s.get("context_hours"),
554
+ "horizon_h": s.get("horizon_hours"),
555
+ "forecast_peak_m": s.get("forecast_peak_m"),
556
+ "forecast_peak_hours_ahead": s.get("forecast_peak_hours_ahead"),
557
+ "interesting": s.get("interesting"),
558
+ }
559
+ return state.update(ttm_battery_surge=s, trace=trace)
560
+ except Exception as e:
561
+ rec["ok"] = False; rec["err"] = str(e)
562
+ log.exception("ttm_battery_surge failed")
563
+ return state.update(ttm_battery_surge=None, trace=trace)
564
+ finally:
565
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
566
+
567
+
568
+ @action(reads=["lat", "lon"], writes=["floodnet_forecast", "trace"])
569
+ def step_floodnet_forecast(state: State) -> State:
570
+ """TTM r2 forecast of flood-event recurrence at the nearest FloodNet
571
+ sensor. Reuses the same (512, 96) singleton as ttm_311_forecast β€” no
572
+ additional model loaded into memory. Silent when the sensor has too
573
+ few historical events for a defensible forecast."""
574
+ rec, trace = _step(state, "floodnet_forecast")
575
+ try:
576
+ if state.get("lat") is None:
577
+ rec["ok"] = False; rec["err"] = "no coords"
578
+ return state.update(floodnet_forecast=None, trace=trace)
579
+ if not _in_nyc(state["lat"], state["lon"]):
580
+ rec["ok"] = False; rec["err"] = "out of NYC scope"
581
+ return state.update(floodnet_forecast=None, trace=trace)
582
+ s = fn_forecast.summary_for_point(state["lat"], state["lon"])
583
+ rec["ok"] = bool(s.get("available"))
584
+ if not rec["ok"]:
585
+ rec["err"] = s.get("reason", "unavailable")
586
+ else:
587
+ rec["result"] = {
588
+ "sensor_id": s.get("sensor_id"),
589
+ "distance_m": s.get("distance_from_query_m"),
590
+ "history_28d": s.get("history_recent_28d_events"),
591
+ "forecast_28d": s.get("forecast_28d_expected_events"),
592
+ "accelerating": s.get("accelerating"),
593
+ }
594
+ return state.update(floodnet_forecast=s if rec["ok"] else None,
595
+ trace=trace)
596
+ except Exception as e:
597
+ rec["ok"] = False; rec["err"] = str(e)
598
+ log.exception("floodnet_forecast failed")
599
+ return state.update(floodnet_forecast=None, trace=trace)
600
+ finally:
601
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
602
+
603
+
604
+ @action(reads=["lat", "lon"], writes=["npcc4_slr", "trace"])
605
+ def step_npcc4_projection(state: State) -> State:
606
+ """NPCC4 (2024) sea-level rise table β€” static lookup, always available."""
607
+ rec, trace = _step(state, "npcc4_projection")
608
+ try:
609
+ s = npcc4_slr.get_projections()
610
+ rec["ok"] = True
611
+ rec["result"] = {
612
+ "2050_10th_in": s["2050"]["10"]["in"],
613
+ "2050_50th_in": s["2050"]["50"]["in"],
614
+ "2050_90th_in": s["2050"]["90"]["in"],
615
+ "2100_90th_in": s["2100"]["90"]["in"],
616
+ }
617
+ return state.update(npcc4_slr=s, trace=trace)
618
+ except Exception as e:
619
+ rec["ok"] = False; rec["err"] = str(e)
620
+ log.exception("npcc4_projection failed")
621
+ return state.update(npcc4_slr=None, trace=trace)
622
+ finally:
623
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
624
+
625
+
626
+ @action(reads=["lat", "lon"], writes=["mta_entrances", "trace"])
627
+ def step_mta_entrances(state: State) -> State:
628
+ rec, trace = _step(state, "mta_entrance_exposure")
629
+ try:
630
+ if state.get("lat") is None:
631
+ rec["ok"] = False; rec["err"] = "no coords"
632
+ return state.update(mta_entrances=None, trace=trace)
633
+ if not _in_nyc(state["lat"], state["lon"]):
634
+ rec["ok"] = False; rec["err"] = "out of NYC scope"
635
+ return state.update(mta_entrances=None, trace=trace)
636
+ s = r_mta.summary_for_point(state["lat"], state["lon"])
637
+ if not s.get("available"):
638
+ rec["ok"] = False; rec["err"] = "no entrances within radius"
639
+ return state.update(mta_entrances=None, trace=trace)
640
+ rec["ok"] = True
641
+ rec["result"] = {
642
+ "n_entrances": s["n_entrances"],
643
+ "n_inside_sandy_2012": s["n_inside_sandy_2012"],
644
+ "n_in_dep_extreme_2080": s["n_in_dep_extreme_2080"],
645
+ }
646
+ return state.update(mta_entrances=s, trace=trace)
647
+ except Exception as e:
648
+ rec["ok"] = False; rec["err"] = str(e)
649
+ log.exception("mta_entrances failed")
650
+ return state.update(mta_entrances=None, trace=trace)
651
+ finally:
652
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
653
+
654
+
655
+ @action(reads=["lat", "lon"], writes=["nycha_developments", "trace"])
656
+ def step_nycha(state: State) -> State:
657
+ rec, trace = _step(state, "nycha_development_exposure")
658
+ try:
659
+ if state.get("lat") is None:
660
+ rec["ok"] = False; rec["err"] = "no coords"
661
+ return state.update(nycha_developments=None, trace=trace)
662
+ if not _in_nyc(state["lat"], state["lon"]):
663
+ rec["ok"] = False; rec["err"] = "out of NYC scope"
664
+ return state.update(nycha_developments=None, trace=trace)
665
+ s = r_nycha.summary_for_point(state["lat"], state["lon"])
666
+ if not s.get("available"):
667
+ rec["ok"] = False; rec["err"] = "no NYCHA developments within radius"
668
+ return state.update(nycha_developments=None, trace=trace)
669
+ rec["ok"] = True
670
+ rec["result"] = {
671
+ "n_developments": s["n_developments"],
672
+ "n_inside_sandy_2012": s["n_inside_sandy_2012"],
673
+ "n_in_dep_extreme_2080": s["n_in_dep_extreme_2080"],
674
+ }
675
+ return state.update(nycha_developments=s, trace=trace)
676
+ except Exception as e:
677
+ rec["ok"] = False; rec["err"] = str(e)
678
+ log.exception("nycha failed")
679
+ return state.update(nycha_developments=None, trace=trace)
680
+ finally:
681
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
682
+
683
+
684
+ @action(reads=["lat", "lon"], writes=["doe_schools", "trace"])
685
+ def step_doe_schools(state: State) -> State:
686
+ rec, trace = _step(state, "doe_school_exposure")
687
+ try:
688
+ if state.get("lat") is None:
689
+ rec["ok"] = False; rec["err"] = "no coords"
690
+ return state.update(doe_schools=None, trace=trace)
691
+ if not _in_nyc(state["lat"], state["lon"]):
692
+ rec["ok"] = False; rec["err"] = "out of NYC scope"
693
+ return state.update(doe_schools=None, trace=trace)
694
+ s = r_schools.summary_for_point(state["lat"], state["lon"])
695
+ if not s.get("available"):
696
+ rec["ok"] = False; rec["err"] = "no schools within radius"
697
+ return state.update(doe_schools=None, trace=trace)
698
+ rec["ok"] = True
699
+ rec["result"] = {
700
+ "n_schools": s["n_schools"],
701
+ "n_inside_sandy_2012": s["n_inside_sandy_2012"],
702
+ "n_in_dep_extreme_2080": s["n_in_dep_extreme_2080"],
703
+ }
704
+ return state.update(doe_schools=s, trace=trace)
705
+ except Exception as e:
706
+ rec["ok"] = False; rec["err"] = str(e)
707
+ log.exception("doe_schools failed")
708
+ return state.update(doe_schools=None, trace=trace)
709
+ finally:
710
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
711
+
712
+
713
+ @action(reads=["lat", "lon"], writes=["doh_hospitals", "trace"])
714
+ def step_doh_hospitals(state: State) -> State:
715
+ rec, trace = _step(state, "doh_hospital_exposure")
716
+ try:
717
+ if state.get("lat") is None:
718
+ rec["ok"] = False; rec["err"] = "no coords"
719
+ return state.update(doh_hospitals=None, trace=trace)
720
+ if not _in_nyc(state["lat"], state["lon"]):
721
+ rec["ok"] = False; rec["err"] = "out of NYC scope"
722
+ return state.update(doh_hospitals=None, trace=trace)
723
+ s = r_hospitals.summary_for_point(state["lat"], state["lon"])
724
+ if not s.get("available"):
725
+ rec["ok"] = False; rec["err"] = "no hospitals within radius"
726
+ return state.update(doh_hospitals=None, trace=trace)
727
+ rec["ok"] = True
728
+ rec["result"] = {
729
+ "n_hospitals": s["n_hospitals"],
730
+ "n_inside_sandy_2012": s["n_inside_sandy_2012"],
731
+ "n_in_dep_extreme_2080": s["n_in_dep_extreme_2080"],
732
+ }
733
+ return state.update(doh_hospitals=s, trace=trace)
734
+ except Exception as e:
735
+ rec["ok"] = False; rec["err"] = str(e)
736
+ log.exception("doh_hospitals failed")
737
+ return state.update(doh_hospitals=None, trace=trace)
738
+ finally:
739
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
740
+
741
+
742
+ @action(reads=["lat", "lon"], writes=["microtopo", "trace"])
743
+ def step_microtopo(state: State) -> State:
744
+ rec, trace = _step(state, "microtopo_lidar")
745
+ try:
746
+ if state.get("lat") is None:
747
+ rec["ok"] = False; rec["err"] = "no coords"
748
+ return state.update(microtopo=None, trace=trace)
749
+ if not _in_nyc(state["lat"], state["lon"]):
750
+ rec["ok"] = False; rec["err"] = "out of NYC scope"
751
+ return state.update(microtopo=None, trace=trace)
752
+ m = microtopo.microtopo_at(state["lat"], state["lon"])
753
+ if m is None:
754
+ rec["ok"] = False; rec["err"] = "DEM fetch failed"
755
+ return state.update(microtopo=None, trace=trace)
756
+ rec["ok"] = True
757
+ rec["result"] = {
758
+ "elev_m": m.point_elev_m,
759
+ "pct_200m": m.rel_elev_pct_200m,
760
+ "relief_m": m.basin_relief_m,
761
+ }
762
+ return state.update(microtopo=vars(m), trace=trace)
763
+ except Exception as e:
764
+ rec["ok"] = False; rec["err"] = str(e)
765
+ log.exception("microtopo failed")
766
+ return state.update(microtopo=None, trace=trace)
767
+ finally:
768
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
769
+
770
+
771
+
772
+
773
+ @action(reads=["lat", "lon"], writes=["eo_chip", "trace"])
774
+ def step_eo_chip(state: State) -> State:
775
+ """Fetch one S2L2A + S1RTC + DEM chip per query and stash it in
776
+ state for the TerraMind-NYC specialists.
777
+
778
+ Centralised so step_terramind_lulc and step_terramind_buildings
779
+ don't each re-fetch ~150 MB of imagery. Best-effort by design β€”
780
+ a deps-missing or no-scene outcome writes `{ok: False, skipped: ...}`
781
+ and the downstream TerraMind specialists silently no-op."""
782
+ rec, trace = _step(state, "eo_chip_fetch")
783
+ try:
784
+ if state.get("lat") is None:
785
+ rec["ok"] = False; rec["err"] = "no coords"
786
+ return state.update(eo_chip=None, trace=trace)
787
+ if not _in_nyc(state["lat"], state["lon"]):
788
+ rec["ok"] = False; rec["err"] = "out of NYC scope"
789
+ return state.update(eo_chip=None, trace=trace)
790
+ from app.context import eo_chip_cache
791
+ chip = eo_chip_cache.fetch(state["lat"], state["lon"])
792
+ rec["ok"] = bool(chip.get("ok"))
793
+ if not rec["ok"]:
794
+ rec["err"] = chip.get("skipped") or chip.get("err") or "unavailable"
795
+ else:
796
+ rec["result"] = {
797
+ "scene_id": (chip.get("s2_meta") or {}).get("scene_id"),
798
+ "scene_date": ((chip.get("s2_meta") or {}).get("datetime") or "")[:10],
799
+ "cloud_cover": (chip.get("s2_meta") or {}).get("cloud_cover"),
800
+ "has_s1": chip.get("s1") is not None,
801
+ "has_dem": chip.get("dem") is not None,
802
+ }
803
+ return state.update(eo_chip=chip, trace=trace)
804
+ except Exception as e:
805
+ rec["ok"] = False; rec["err"] = str(e)
806
+ log.exception("eo_chip failed")
807
+ return state.update(eo_chip=None, trace=trace)
808
+ finally:
809
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
810
+
811
+
812
+ @action(reads=["lat", "lon", "eo_chip"], writes=["terramind_lulc", "trace"])
813
+ def step_terramind_lulc(state: State) -> State:
814
+ """5-class macro NYC LULC via msradam/TerraMind-NYC-Adapters.
815
+
816
+ Consumes the shared chip from step_eo_chip; if that didn't fire
817
+ cleanly this no-ops. Adapter loading (~1.6 GB base + ~325 MB LoRA)
818
+ is lazy on first call and cached across queries."""
819
+ rec, trace = _step(state, "terramind_lulc")
820
+ try:
821
+ if state.get("lat") is None:
822
+ rec["ok"] = False; rec["err"] = "no coords"
823
+ return state.update(terramind_lulc=None, trace=trace)
824
+ if not _in_nyc(state["lat"], state["lon"]):
825
+ rec["ok"] = False; rec["err"] = "out of NYC scope"
826
+ return state.update(terramind_lulc=None, trace=trace)
827
+ chip = state.get("eo_chip") or {}
828
+ if not chip.get("ok"):
829
+ rec["ok"] = False
830
+ rec["err"] = chip.get("skipped") or chip.get("err") or "no chip"
831
+ return state.update(terramind_lulc=None, trace=trace)
832
+ from app.context import terramind_nyc
833
+ tensors = chip.get("tensors") or {}
834
+ out = terramind_nyc.lulc(
835
+ tensors.get("S2L2A"),
836
+ s1rtc=tensors.get("S1RTC"),
837
+ dem=tensors.get("DEM"),
838
+ bounds_4326=chip.get("bounds_4326"),
839
+ )
840
+ rec["ok"] = bool(out.get("ok"))
841
+ if not rec["ok"]:
842
+ rec["err"] = out.get("skipped") or out.get("err") or "unavailable"
843
+ else:
844
+ rec["result"] = {
845
+ "dominant_class": out.get("dominant_class"),
846
+ "dominant_pct": out.get("dominant_pct"),
847
+ "n_classes_observed": len(out.get("class_fractions") or {}),
848
+ }
849
+ return state.update(terramind_lulc=out, trace=trace)
850
+ except Exception as e:
851
+ rec["ok"] = False; rec["err"] = str(e)
852
+ log.exception("terramind_lulc failed")
853
+ return state.update(terramind_lulc=None, trace=trace)
854
+ finally:
855
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
856
+
857
+
858
+ @action(reads=["lat", "lon", "eo_chip"],
859
+ writes=["terramind_buildings", "trace"])
860
+ def step_terramind_buildings(state: State) -> State:
861
+ """Binary NYC building-footprint mask via msradam/TerraMind-NYC-Adapters."""
862
+ rec, trace = _step(state, "terramind_buildings")
863
+ try:
864
+ if state.get("lat") is None:
865
+ rec["ok"] = False; rec["err"] = "no coords"
866
+ return state.update(terramind_buildings=None, trace=trace)
867
+ if not _in_nyc(state["lat"], state["lon"]):
868
+ rec["ok"] = False; rec["err"] = "out of NYC scope"
869
+ return state.update(terramind_buildings=None, trace=trace)
870
+ chip = state.get("eo_chip") or {}
871
+ if not chip.get("ok"):
872
+ rec["ok"] = False
873
+ rec["err"] = chip.get("skipped") or chip.get("err") or "no chip"
874
+ return state.update(terramind_buildings=None, trace=trace)
875
+ from app.context import terramind_nyc
876
+ tensors = chip.get("tensors") or {}
877
+ out = terramind_nyc.buildings(
878
+ tensors.get("S2L2A"),
879
+ s1rtc=tensors.get("S1RTC"),
880
+ dem=tensors.get("DEM"),
881
+ bounds_4326=chip.get("bounds_4326"),
882
+ )
883
+ rec["ok"] = bool(out.get("ok"))
884
+ if not rec["ok"]:
885
+ rec["err"] = out.get("skipped") or out.get("err") or "unavailable"
886
+ else:
887
+ rec["result"] = {
888
+ "pct_buildings": out.get("pct_buildings"),
889
+ "n_building_components": out.get("n_building_components"),
890
+ }
891
+ return state.update(terramind_buildings=out, trace=trace)
892
+ except Exception as e:
893
+ rec["ok"] = False; rec["err"] = str(e)
894
+ log.exception("terramind_buildings failed")
895
+ return state.update(terramind_buildings=None, trace=trace)
896
+ finally:
897
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
898
+
899
+
900
+ @action(reads=["geocode", "sandy", "dep", "floodnet", "nyc311", "microtopo",
901
+ "ida_hwm", "prithvi_water", "noaa_tides", "nws_alerts", "nws_obs",
902
+ "ttm_forecast"],
903
+ writes=["rag", "trace"])
904
+ def step_rag(state: State) -> State:
905
+ rec, trace = _step(state, "rag_granite_embedding")
906
+ try:
907
+ geo = state.get("geocode") or {}
908
+ if not _in_nyc(geo.get("lat"), geo.get("lon")):
909
+ rec["ok"] = False; rec["err"] = "out of NYC scope"
910
+ return state.update(rag=[], trace=trace)
911
+ sandy = state.get("sandy")
912
+ dep = state.get("dep") or {}
913
+ # Build a context-rich query so retrieval pulls policy paragraphs
914
+ # relevant to *this* address, not generic flood text.
915
+ bits = []
916
+ if geo.get("address"):
917
+ bits.append(f"address {geo['address']}")
918
+ if geo.get("borough"):
919
+ bits.append(f"in {geo['borough']}")
920
+ if sandy:
921
+ bits.append("inside Hurricane Sandy 2012 inundation zone")
922
+ for v in dep.values():
923
+ if v.get("depth_class", 0) > 0:
924
+ bits.append(f"in {v['depth_label']} pluvial scenario")
925
+ bits.append("flood resilience plan, vulnerability, hardening, mitigation")
926
+ q = "; ".join(bits)
927
+ hits = rag_retrieve(q, k=3, min_score=0.45)
928
+ rec["ok"] = True
929
+ rec["result"] = {"hits": len(hits),
930
+ "top": [(h["doc_id"], round(h["score"], 2)) for h in hits]}
931
+ return state.update(rag=hits, trace=trace)
932
+ except Exception as e:
933
+ rec["ok"] = False; rec["err"] = str(e)
934
+ log.exception("rag failed")
935
+ return state.update(rag=[], trace=trace)
936
+ finally:
937
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
938
+
939
+
940
+ @action(reads=["rag"], writes=["gliner", "trace"])
941
+ def step_gliner(state: State) -> State:
942
+ """GLiNER typed-entity extraction over the top RAG paragraphs.
943
+
944
+ Adds structured fields (`agency`, `dollar_amount`,
945
+ `infrastructure_project`, `nyc_location`, `date_range`) the
946
+ reconciler can cite with `[gliner_<source>]`. Silent no-op when
947
+ disabled via RIPRAP_GLINER_ENABLE=0 or when the model failed to
948
+ load β€” preserves the existing FSM contract.
949
+ """
950
+ rec, trace = _step(state, "gliner_extract")
951
+ try:
952
+ from app.context.gliner_extract import extract_for_rag_hits
953
+ hits = state.get("rag") or []
954
+ if not hits:
955
+ rec["ok"] = True
956
+ rec["result"] = {"sources": 0, "skipped": "no rag hits"}
957
+ return state.update(gliner={}, trace=trace)
958
+ out = extract_for_rag_hits(hits)
959
+ rec["ok"] = True
960
+ rec["result"] = {
961
+ "sources": len(out),
962
+ "totals_by_label": _label_counts(out),
963
+ }
964
+ return state.update(gliner=out, trace=trace)
965
+ except Exception as e:
966
+ rec["ok"] = False
967
+ rec["err"] = str(e)
968
+ log.exception("gliner failed")
969
+ return state.update(gliner={}, trace=trace)
970
+ finally:
971
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
972
+
973
+
974
+ def _label_counts(gliner_out: dict[str, dict]) -> dict[str, int]:
975
+ counts: dict[str, int] = {}
976
+ for src in gliner_out.values():
977
+ for e in src.get("entities", []):
978
+ counts[e["label"]] = counts.get(e["label"], 0) + 1
979
+ return counts
980
+
981
+
982
+ @action(reads=["geocode", "sandy", "dep", "floodnet", "nyc311", "microtopo",
983
+ "ida_hwm", "prithvi_water", "prithvi_live", "terramind",
984
+ "terramind_lulc", "terramind_buildings",
985
+ "noaa_tides", "nws_alerts", "nws_obs", "ttm_forecast",
986
+ "ttm_311_forecast", "floodnet_forecast", "npcc4_slr",
987
+ "ttm_battery_surge",
988
+ "mta_entrances",
989
+ "nycha_developments", "doe_schools", "doh_hospitals",
990
+ "rag", "gliner"],
991
+ writes=["paragraph", "audit", "mellea", "citations", "trace"])
992
+ def step_reconcile(state: State) -> State:
993
+ is_strict = _current_strict_mode()
994
+ rec, trace = _step(state, "mellea_reconcile_address" if is_strict else "reconcile_granite41")
995
+ mellea_meta = None
996
+ try:
997
+ snap = {
998
+ "geocode": state.get("geocode"),
999
+ "sandy": state.get("sandy"),
1000
+ "dep": state.get("dep"),
1001
+ "floodnet": state.get("floodnet"),
1002
+ "nyc311": state.get("nyc311"),
1003
+ "microtopo": state.get("microtopo"),
1004
+ "ida_hwm": state.get("ida_hwm"),
1005
+ "prithvi_water": state.get("prithvi_water"),
1006
+ "noaa_tides": state.get("noaa_tides"),
1007
+ "nws_alerts": state.get("nws_alerts"),
1008
+ "nws_obs": state.get("nws_obs"),
1009
+ "ttm_forecast": state.get("ttm_forecast"),
1010
+ "ttm_311_forecast": state.get("ttm_311_forecast"),
1011
+ "floodnet_forecast": state.get("floodnet_forecast"),
1012
+ "npcc4_slr": state.get("npcc4_slr"),
1013
+ "ttm_battery_surge": state.get("ttm_battery_surge"),
1014
+ "rag": state.get("rag"),
1015
+ "gliner": state.get("gliner"),
1016
+ "prithvi_live": state.get("prithvi_live"),
1017
+ "terramind": state.get("terramind"),
1018
+ "terramind_lulc": state.get("terramind_lulc"),
1019
+ "terramind_buildings": state.get("terramind_buildings"),
1020
+ "mta_entrances": state.get("mta_entrances"),
1021
+ "nycha_developments": state.get("nycha_developments"),
1022
+ "doe_schools": state.get("doe_schools"),
1023
+ "doh_hospitals": state.get("doh_hospitals"),
1024
+ }
1025
+ if is_strict:
1026
+ from app.framing import augment_system_prompt
1027
+ from app.mellea_validator import DEFAULT_LOOP_BUDGET, reconcile_strict_streaming
1028
+ from app.reconcile import EXTRA_SYSTEM_PROMPT, build_documents, trim_docs_to_plan
1029
+ doc_msgs = build_documents(snap)
1030
+ doc_msgs = trim_docs_to_plan(doc_msgs, _current_planned_specialists())
1031
+ if not doc_msgs:
1032
+ para = "No grounded data available for this address."
1033
+ audit = {"raw": para, "dropped": []}
1034
+ else:
1035
+ token_cb = _current_token_callback()
1036
+ attempt_cb = _current_mellea_attempt_callback()
1037
+ framed_prompt = augment_system_prompt(
1038
+ EXTRA_SYSTEM_PROMPT,
1039
+ query=_current_user_query() or state.get("query") or "",
1040
+ intent=_current_planner_intent() or "single_address",
1041
+ )
1042
+ # Forward the (delta, attempt_idx) pair through. Older
1043
+ # token_cb signatures were single-arg; we detect by
1044
+ # introspecting the callable's expected positional count
1045
+ # so single_address.py's old shape still works while new
1046
+ # callbacks see the attempt index they need to clear the
1047
+ # frontend buffer on a Mellea reroll.
1048
+ def _fwd_token(delta: str, attempt_idx: int) -> None:
1049
+ if token_cb is None:
1050
+ return
1051
+ try:
1052
+ token_cb(delta, attempt_idx)
1053
+ except TypeError:
1054
+ token_cb(delta)
1055
+ mres = reconcile_strict_streaming(
1056
+ doc_msgs, framed_prompt,
1057
+ user_prompt="Write the cited paragraph now.",
1058
+ loop_budget=DEFAULT_LOOP_BUDGET,
1059
+ on_token=_fwd_token if token_cb else None,
1060
+ on_attempt_end=attempt_cb,
1061
+ )
1062
+ para = mres["paragraph"]
1063
+ audit = {"raw": para, "dropped": []}
1064
+ mellea_meta = {
1065
+ "rerolls": mres["rerolls"],
1066
+ "n_attempts": mres["n_attempts"],
1067
+ "requirements_passed": mres["requirements_passed"],
1068
+ "requirements_failed": mres["requirements_failed"],
1069
+ "requirements_total": mres["requirements_total"],
1070
+ "model": mres["model"],
1071
+ "loop_budget": mres["loop_budget"],
1072
+ }
1073
+ rec["result"] = {
1074
+ "rerolls": (mellea_meta or {}).get("rerolls"),
1075
+ "passed": (f"{len((mellea_meta or {}).get('requirements_passed') or [])}/"
1076
+ f"{(mellea_meta or {}).get('requirements_total') or 0}"),
1077
+ "paragraph_chars": len(para),
1078
+ }
1079
+ else:
1080
+ para, audit = run_reconcile(snap, return_audit=True,
1081
+ on_token=_current_token_callback())
1082
+ rec["result"] = {
1083
+ "paragraph_chars": len(para),
1084
+ "dropped_sentences": len(audit["dropped"]),
1085
+ }
1086
+ # Build citation metadata list from whichever doc_msgs were used.
1087
+ from app.reconcile import build_documents, trim_docs_to_plan
1088
+ _cite_msgs = build_documents(snap)
1089
+ _cite_msgs = trim_docs_to_plan(_cite_msgs, _current_planned_specialists())
1090
+ cite_list = citations_from_docs(_cite_msgs)
1091
+ rec["ok"] = True
1092
+ return state.update(paragraph=para, audit=audit,
1093
+ mellea=mellea_meta, citations=cite_list, trace=trace)
1094
+ except Exception as e:
1095
+ rec["ok"] = False; rec["err"] = str(e)
1096
+ log.exception("reconcile failed")
1097
+ return state.update(paragraph="", audit={"raw": "", "dropped": []},
1098
+ mellea=None, citations=[], trace=trace)
1099
+ finally:
1100
+ rec["elapsed_s"] = round(time.time() - rec["started_at"], 2)
1101
+
1102
+
1103
+ import os as _os # noqa: E402
1104
+
1105
+
1106
+ # Specialists that involve large spatial joins (every NYCHA development
1107
+ # overlapped against multiple flood layers, every DOE school footprint
1108
+ # joined to DEM/HAND, etc.) or per-query model inference (Prithvi-EO live
1109
+ # STAC + ViT, TerraMind diffusion). They're ~1-3 minutes apiece on a
1110
+ # laptop on the FIRST call (the lru_caches inside the registers warm up
1111
+ # afterwards). The previous parallel-fan-out FSM hid that cost behind
1112
+ # the longest single specialist; the linear FSM exposes it.
1113
+ #
1114
+ # Default OFF on local-Ollama so the demo briefing returns in well under
1115
+ # 90 s. Enable explicitly with RIPRAP_HEAVY_SPECIALISTS=1 (e.g. on the
1116
+ # AMD-vLLM path, where the reconciler's ~5 s leaves room for the joins).
1117
+ #
1118
+ # Remote ML lift: when RIPRAP_ML_BACKEND=remote (or auto with a base URL
1119
+ # set) the heavy specialists' GPU work runs on the droplet, so the local
1120
+ # wall-clock cost drops from ~60 s to ~5 s. Default ON in that case so
1121
+ # the public demo never silently disables them.
1122
+ def _remote_ml_configured() -> bool:
1123
+ backend = _os.environ.get("RIPRAP_ML_BACKEND", "auto").lower()
1124
+ if backend == "local":
1125
+ return False
1126
+ return bool(_os.environ.get("RIPRAP_ML_BASE_URL", "").strip())
1127
+
1128
+
1129
+ _HEAVY_DEFAULT = (
1130
+ "1" if (
1131
+ _os.environ.get("RIPRAP_LLM_PRIMARY", "ollama").lower() != "ollama"
1132
+ or _remote_ml_configured()
1133
+ ) else "0"
1134
+ )
1135
+ _HEAVY_SPECIALISTS_ENABLED = _os.environ.get(
1136
+ "RIPRAP_HEAVY_SPECIALISTS", _HEAVY_DEFAULT,
1137
+ ).lower() in ("1", "true", "yes")
1138
+
1139
+ # NYCHA / DOE / DOH registers load a 91 MB sandy_inundation.geojson via
1140
+ # geopandas on first call. On machines with slow I/O or single-threaded
1141
+ # Python GIL contention (M3 local dev) this takes 3–5 min and makes the
1142
+ # first single_address query appear hung. Disable by default; enable on
1143
+ # the AMD droplet where the server pre-warms these at startup.
1144
+ _NYCHA_REGISTERS_ENABLED = _os.environ.get(
1145
+ "RIPRAP_NYCHA_REGISTERS", "0",
1146
+ ).lower() in ("1", "true", "yes")
1147
+
1148
+
1149
+ def build_app(query: str):
1150
+ """Linear, single-action-per-step Burr application.
1151
+
1152
+ Order: cheap-first geo + flood layers, then live live network signals,
1153
+ then RAG β†’ reconcile. Heavy specialists (NYCHA / DOE / DOH register
1154
+ joins, Prithvi-EO live STAC, TerraMind diffusion) are gated behind
1155
+ RIPRAP_HEAVY_SPECIALISTS β€” see the module-level note above.
1156
+ """
1157
+ builder = (
1158
+ ApplicationBuilder()
1159
+ .with_state(query=query, trace=[])
1160
+ .with_entrypoint("geocode")
1161
+ )
1162
+
1163
+ actions: dict[str, Any] = {
1164
+ "geocode": step_geocode,
1165
+ "sandy": step_sandy,
1166
+ "dep": step_dep,
1167
+ "floodnet": step_floodnet,
1168
+ "nyc311": step_311,
1169
+ "noaa_tides": step_noaa_tides,
1170
+ "nws_alerts": step_nws_alerts,
1171
+ "nws_obs": step_nws_obs,
1172
+ "ttm_forecast": step_ttm_forecast,
1173
+ "ttm_311_forecast": step_ttm_311_forecast,
1174
+ "floodnet_forecast": step_floodnet_forecast,
1175
+ "npcc4_projection": step_npcc4_projection,
1176
+ "ttm_battery_surge": step_ttm_battery_surge,
1177
+ "microtopo": step_microtopo,
1178
+ "ida_hwm": step_ida_hwm,
1179
+ "mta_entrances": step_mta_entrances,
1180
+ "prithvi": step_prithvi, # baked GeoJSON polygons for Ida; cheap
1181
+ }
1182
+ if _HEAVY_SPECIALISTS_ENABLED and _NYCHA_REGISTERS_ENABLED:
1183
+ actions["nycha"] = step_nycha
1184
+ actions["doe_schools"] = step_doe_schools
1185
+ actions["doh_hospitals"] = step_doh_hospitals
1186
+ if _HEAVY_SPECIALISTS_ENABLED:
1187
+ actions["prithvi_live"] = step_prithvi_live
1188
+ actions["terramind"] = step_terramind
1189
+ # New TerraMind-NYC LoRA family β€” one chip fetch feeds two
1190
+ # specialists. Keep eo_chip directly before the two consumers
1191
+ # so the chip stays warm in memory and isn't garbage-collected
1192
+ # by anything in between.
1193
+ actions["eo_chip"] = step_eo_chip
1194
+ actions["terramind_lulc"] = step_terramind_lulc
1195
+ actions["terramind_buildings"] = step_terramind_buildings
1196
+ actions["rag"] = step_rag
1197
+ actions["gliner"] = step_gliner
1198
+ actions["reconcile"] = step_reconcile
1199
+
1200
+ # Sequential transitions β€” pair every adjacent action in the dict order.
1201
+ keys = list(actions.keys())
1202
+ transitions = list(zip(keys, keys[1:]))
1203
+
1204
+ return (
1205
+ builder.with_actions(**actions).with_transitions(*transitions).build()
1206
+ )
1207
+
1208
+
1209
+ def _summarize_energy(trace: list) -> dict | None:
1210
+ rec_step = next((t for t in trace if t.get("step") == "reconcile_granite41"
1211
+ and t.get("ok")), None)
1212
+ if not rec_step:
1213
+ return None
1214
+ total_s = sum(t.get("elapsed_s", 0) or 0 for t in trace)
1215
+ return energy_estimate(rec_step.get("elapsed_s", 0) or 0, total_s)
1216
+
1217
+
1218
+ def _summarize_emissions() -> dict | None:
1219
+ """Snapshot the active per-call emissions tracker, if installed.
1220
+
1221
+ Returns None when no tracker is bound to this thread (e.g. unit
1222
+ tests that call `fsm.run` directly without going through the
1223
+ web/intent layer that installs one)."""
1224
+ t = emissions.current()
1225
+ return t.summarize() if t is not None else None
1226
+
1227
+
1228
+ def run(query: str) -> dict[str, Any]:
1229
+ app = build_app(query)
1230
+ final_action, _, final_state = app.run(halt_after=["reconcile"])
1231
+ trace = final_state.get("trace", [])
1232
+ return {
1233
+ "query": query,
1234
+ "geocode": final_state.get("geocode"),
1235
+ "sandy": final_state.get("sandy"),
1236
+ "dep": final_state.get("dep"),
1237
+ "floodnet": final_state.get("floodnet"),
1238
+ "nyc311": final_state.get("nyc311"),
1239
+ "microtopo": final_state.get("microtopo"),
1240
+ "ida_hwm": final_state.get("ida_hwm"),
1241
+ "prithvi_water": final_state.get("prithvi_water"),
1242
+ "terramind": final_state.get("terramind"),
1243
+ "terramind_lulc": final_state.get("terramind_lulc"),
1244
+ "terramind_buildings": final_state.get("terramind_buildings"),
1245
+ "eo_chip": final_state.get("eo_chip"),
1246
+ "noaa_tides": final_state.get("noaa_tides"),
1247
+ "nws_alerts": final_state.get("nws_alerts"),
1248
+ "nws_obs": final_state.get("nws_obs"),
1249
+ "ttm_forecast": final_state.get("ttm_forecast"),
1250
+ "ttm_311_forecast": final_state.get("ttm_311_forecast"),
1251
+ "floodnet_forecast": final_state.get("floodnet_forecast"),
1252
+ "ttm_battery_surge": final_state.get("ttm_battery_surge"),
1253
+ "mta_entrances": final_state.get("mta_entrances"),
1254
+ "nycha_developments": final_state.get("nycha_developments"),
1255
+ "doe_schools": final_state.get("doe_schools"),
1256
+ "doh_hospitals": final_state.get("doh_hospitals"),
1257
+ "rag": final_state.get("rag"),
1258
+ "paragraph": final_state.get("paragraph"),
1259
+ "audit": final_state.get("audit"),
1260
+ "mellea": final_state.get("mellea"),
1261
+ "energy": _summarize_energy(trace),
1262
+ "emissions": _summarize_emissions(),
1263
+ "trace": trace,
1264
+ }
1265
+
1266
+
1267
+ def iter_steps(query: str):
1268
+ """Yield SSE-friendly events as the FSM runs.
1269
+
1270
+ Each Burr action emits exactly one trace record on completion; we
1271
+ yield it as a `step` event the moment the iterate loop returns from
1272
+ that action. Reconciler tokens stream through the threadlocal
1273
+ `set_token_callback` (installed before this generator is iterated),
1274
+ not through this queue.
1275
+
1276
+ Burr's `app.iterate(halt_after=["reconcile"])` runs synchronously,
1277
+ yielding `(action, result, state)` after every action. We drive it
1278
+ in a background thread so the per-action SSE events reach the
1279
+ client as soon as each action returns, while the reconciler's
1280
+ token callback fires concurrently from the same thread.
1281
+ """
1282
+ import queue
1283
+
1284
+ q: queue.Queue[tuple[str, Any] | None] = queue.Queue()
1285
+ seen_keys: set[tuple[str, float]] = set()
1286
+
1287
+ def _push_step(rec: dict) -> None:
1288
+ key = (rec.get("step", ""), rec.get("started_at", 0.0))
1289
+ if key in seen_keys:
1290
+ return
1291
+ seen_keys.add(key)
1292
+ q.put(("step", rec))
1293
+
1294
+ app = build_app(query)
1295
+ final_state_holder: dict[str, Any] = {}
1296
+
1297
+ # Threadlocals are per-thread; the request thread (single_address.run
1298
+ # / neighborhood.run) sets the strict-mode flag, planner specialist
1299
+ # set, and token / Mellea-attempt callbacks, but Burr's app.iterate
1300
+ # runs in this generator's thread. Snapshot the request-thread state
1301
+ # and re-install on the iterate thread so step_reconcile sees them.
1302
+ _captured_strict = _current_strict_mode()
1303
+ _captured_planned = _current_planned_specialists()
1304
+ _captured_token_cb = _current_token_callback()
1305
+ _captured_mellea_cb = _current_mellea_attempt_callback()
1306
+ _captured_tracker = emissions.current()
1307
+
1308
+ def _run_iterate():
1309
+ set_strict_mode(_captured_strict)
1310
+ set_planned_specialists(_captured_planned)
1311
+ set_token_callback(_captured_token_cb)
1312
+ set_mellea_attempt_callback(_captured_mellea_cb)
1313
+ emissions.install(_captured_tracker)
1314
+ try:
1315
+ for _action_obj, _result, state in app.iterate(halt_after=["reconcile"]):
1316
+ final_state_holder["state"] = state
1317
+ # Each action appends one record to state.trace; emit the
1318
+ # most recent so the SSE client gets the step event the
1319
+ # moment Burr returns from that action.
1320
+ trace = state.get("trace") or []
1321
+ if trace:
1322
+ _push_step(trace[-1])
1323
+ except Exception as e:
1324
+ log.exception("iterate raised")
1325
+ q.put(("error", {"err": f"{type(e).__name__}: {e}"}))
1326
+ finally:
1327
+ set_strict_mode(False)
1328
+ set_planned_specialists(None)
1329
+ set_token_callback(None)
1330
+ set_mellea_attempt_callback(None)
1331
+ emissions.install(None)
1332
+ q.put(None) # sentinel
1333
+
1334
+ runner = _threading.Thread(target=_run_iterate, name="riprap-fsm",
1335
+ daemon=True)
1336
+ runner.start()
1337
+
1338
+ while True:
1339
+ item = q.get()
1340
+ if item is None:
1341
+ break
1342
+ kind, payload = item
1343
+ if kind == "step":
1344
+ yield {
1345
+ "kind": "step",
1346
+ "step": payload.get("step"),
1347
+ "ok": payload.get("ok"),
1348
+ "elapsed_s": payload.get("elapsed_s"),
1349
+ "result": payload.get("result"),
1350
+ "err": payload.get("err"),
1351
+ }
1352
+ elif kind == "error":
1353
+ yield {"kind": "error", **payload}
1354
+
1355
+ runner.join(timeout=5)
1356
+ state = final_state_holder.get("state")
1357
+ if state is None:
1358
+ yield {"kind": "final", "paragraph": "", "error": "FSM failed before any action completed"}
1359
+ return
1360
+ trace = state.get("trace", [])
1361
+ yield {
1362
+ "kind": "final",
1363
+ "geocode": state.get("geocode"),
1364
+ "sandy": state.get("sandy"),
1365
+ "dep": state.get("dep"),
1366
+ "floodnet": state.get("floodnet"),
1367
+ "nyc311": state.get("nyc311"),
1368
+ "microtopo": state.get("microtopo"),
1369
+ "ida_hwm": state.get("ida_hwm"),
1370
+ "prithvi_water": state.get("prithvi_water"),
1371
+ "prithvi_live": state.get("prithvi_live"),
1372
+ "terramind": state.get("terramind"),
1373
+ "terramind_lulc": state.get("terramind_lulc"),
1374
+ "terramind_buildings": state.get("terramind_buildings"),
1375
+ "noaa_tides": state.get("noaa_tides"),
1376
+ "nws_alerts": state.get("nws_alerts"),
1377
+ "nws_obs": state.get("nws_obs"),
1378
+ "ttm_forecast": state.get("ttm_forecast"),
1379
+ "ttm_311_forecast": state.get("ttm_311_forecast"),
1380
+ "floodnet_forecast": state.get("floodnet_forecast"),
1381
+ "ttm_battery_surge": state.get("ttm_battery_surge"),
1382
+ "mta_entrances": state.get("mta_entrances"),
1383
+ "nycha_developments": state.get("nycha_developments"),
1384
+ "doe_schools": state.get("doe_schools"),
1385
+ "doh_hospitals": state.get("doh_hospitals"),
1386
+ "rag": state.get("rag"),
1387
+ "gliner": state.get("gliner"),
1388
+ "paragraph": state.get("paragraph"),
1389
+ "audit": state.get("audit"),
1390
+ "mellea": state.get("mellea"),
1391
+ "citations": state.get("citations"),
1392
+ "energy": _summarize_energy(trace),
1393
+ "emissions": _summarize_emissions(),
1394
+ }
app/geocode.py ADDED
@@ -0,0 +1,138 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Address geocoding β€” NYC primary + national fallback.
2
+
3
+ NYC primary: NYC DCP Geosearch (geosearch.planninglabs.nyc), no auth,
4
+ NYC-only. It will fuzzy-match upstate addresses to NYC streets β€” e.g.
5
+ '257 Washington Ave, Albany NY' silently maps to Clinton Hill, Brooklyn.
6
+ We detect this via a non-NYC region or non-NYC ZIP and fall back to
7
+ OpenStreetMap Nominatim (no key, free, rate-limited per usage policy).
8
+
9
+ Includes a borough-hint post-filter so Queens hyphenated-style addresses
10
+ (e.g. '153-09 90 Ave, Jamaica, Queens') preferentially resolve to the
11
+ borough the user named.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import logging
16
+ import re
17
+ from dataclasses import dataclass
18
+
19
+ import httpx
20
+
21
+ log = logging.getLogger("riprap.geocode")
22
+
23
+ URL = "https://geosearch.planninglabs.nyc/v2/search"
24
+ NOMINATIM_URL = "https://nominatim.openstreetmap.org/search"
25
+ NOMINATIM_UA = "Riprap-NYC/0.5 (civic-flood-tool; +https://huggingface.co/spaces/msradam/riprap-nyc)"
26
+
27
+ # NYC-bbox guard: lat 40.49–40.92, lon -74.27 to -73.69.
28
+ NYC_BBOX = (40.49, -74.27, 40.92, -73.69)
29
+
30
+ _UPSTATE_ZIP_RE = re.compile(r"\b1[2-4]\d{3}\b")
31
+ _BOROUGHS = ("Manhattan", "Bronx", "Brooklyn", "Queens", "Staten Island")
32
+
33
+ def _detect_borough(text: str) -> str | None:
34
+ t = text.lower()
35
+ for b in _BOROUGHS:
36
+ if b.lower() in t:
37
+ return b
38
+ # neighborhood -> borough hints
39
+ hints = {
40
+ "queens": "Queens", "jamaica": "Queens", "rockaway": "Queens",
41
+ "astoria": "Queens", "flushing": "Queens",
42
+ "manhattan": "Manhattan", "harlem": "Manhattan", "soho": "Manhattan",
43
+ "brooklyn": "Brooklyn", "bushwick": "Brooklyn", "red hook": "Brooklyn",
44
+ "bronx": "Bronx", "fordham": "Bronx",
45
+ "staten island": "Staten Island",
46
+ }
47
+ for needle, boro in hints.items():
48
+ if needle in t:
49
+ return boro
50
+ return None
51
+
52
+ @dataclass
53
+ class GeocodeHit:
54
+ address: str
55
+ borough: str | None
56
+ lat: float
57
+ lon: float
58
+ bbl: str | None
59
+ bin: str | None
60
+ raw: dict
61
+
62
+ def geocode(text: str, limit: int = 5) -> list[GeocodeHit]:
63
+ """NYC Geosearch primary."""
64
+ try:
65
+ r = httpx.get(URL, params={"text": text, "size": limit}, timeout=5)
66
+ r.raise_for_status()
67
+ feats = r.json().get("features", [])
68
+ out = []
69
+ for f in feats:
70
+ p = f.get("properties", {})
71
+ coords = (f.get("geometry") or {}).get("coordinates") or [None, None]
72
+ out.append(GeocodeHit(
73
+ address=p.get("label") or p.get("name") or text,
74
+ borough=p.get("borough"),
75
+ lat=coords[1],
76
+ lon=coords[0],
77
+ bbl=p.get("addendum", {}).get("pad", {}).get("bbl"),
78
+ bin=p.get("addendum", {}).get("pad", {}).get("bin"),
79
+ raw=p,
80
+ ))
81
+ return out
82
+ except Exception as e:
83
+ log.warning("Geosearch failed: %r", e)
84
+ return []
85
+
86
+ def geocode_nominatim(text: str) -> GeocodeHit | None:
87
+ """National OSM Nominatim fallback."""
88
+ try:
89
+ r = httpx.get(NOMINATIM_URL, params={
90
+ "q": text, "format": "jsonv2", "addressdetails": "1",
91
+ "limit": 1, "countrycodes": "us",
92
+ }, headers={"User-Agent": NOMINATIM_UA}, timeout=10)
93
+ r.raise_for_status()
94
+ rows = r.json()
95
+ except Exception as e:
96
+ log.warning("Nominatim fetch failed: %r", e)
97
+ return None
98
+ if not rows:
99
+ return None
100
+ row = rows[0]
101
+ addr = row.get("address") or {}
102
+
103
+ # Try to map Nominatim borough/county back to NYC standard
104
+ boro = addr.get("suburb") or addr.get("city_district") or addr.get("county")
105
+ if boro and "Kings" in boro: boro = "Brooklyn"
106
+ if boro and "New York County" in boro: boro = "Manhattan"
107
+ if boro and "Queens" in boro: boro = "Queens"
108
+ if boro and "Bronx" in boro: boro = "Bronx"
109
+ if boro and "Richmond" in boro: boro = "Staten Island"
110
+
111
+ return GeocodeHit(
112
+ address=row.get("display_name") or text,
113
+ borough=boro,
114
+ lat=float(row["lat"]),
115
+ lon=float(row["lon"]),
116
+ bbl=None, # Nominatim doesn't have BBLs
117
+ bin=None,
118
+ raw={"source": "nominatim", **row},
119
+ )
120
+
121
+ def geocode_one(text: str) -> GeocodeHit | None:
122
+ """Dynamic geocoder with failover."""
123
+ # 1. Try Geosearch
124
+ hits = geocode(text)
125
+ hint = _detect_borough(text)
126
+
127
+ if hint:
128
+ in_boro = [h for h in hits if h.borough and h.borough.lower() == hint.lower()]
129
+ if in_boro: return in_boro[0]
130
+
131
+ if hits:
132
+ top = hits[0]
133
+ if top.lat and 40.4 <= top.lat <= 41.0: # Broad NYC check
134
+ return top
135
+
136
+ # 2. Fall back to Nominatim
137
+ log.info("Falling back to Nominatim for %r", text)
138
+ return geocode_nominatim(text)
app/inference.py ADDED
@@ -0,0 +1,268 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Remote-vs-local ML inference router.
2
+
3
+ Mirrors the call-surface shape of `app/llm.py` but for the non-LLM
4
+ heavy models (Prithvi, TerraMind, TTM, Granite Embedding, GLiNER).
5
+
6
+ The droplet runs a `riprap-models` FastAPI service alongside vLLM that
7
+ exposes an OpenAI-style endpoint per model class. When configured the
8
+ router POSTs the relevant payload there and returns the parsed response;
9
+ on connection error / 5xx / timeout it surfaces a typed exception that
10
+ caller modules catch and fall back to a local in-process model load.
11
+
12
+ Backend selection (env):
13
+
14
+ RIPRAP_ML_BACKEND = "remote" | "local" | "auto" (default: auto)
15
+ - remote: use only the droplet, raise if it errors
16
+ - local : never call the droplet, always use the
17
+ in-process model
18
+ - auto : try remote first, fall back to local if
19
+ remote is unreachable / errors out;
20
+ same semantics as app/llm.py
21
+ RIPRAP_ML_BASE_URL = http://129.212.181.238:8002 (no trailing slash)
22
+ RIPRAP_ML_API_KEY = <bearer token>
23
+
24
+ The router is *transport*-only β€” it does not own model bytes, weights,
25
+ or framework imports. Each specialist that wants remote inference calls
26
+ into the helpers below and provides its own local fallback. That keeps
27
+ the dependency graph clean: the local code path keeps working when the
28
+ RIPRAP_ML_* env is unset (e.g. on first-light dev or in unit tests).
29
+ """
30
+ from __future__ import annotations
31
+
32
+ import base64
33
+ import logging
34
+ import os
35
+ import time
36
+ from collections.abc import Iterable
37
+ from typing import Any
38
+
39
+ import httpx
40
+
41
+ from app import emissions
42
+
43
+ log = logging.getLogger("riprap.inference")
44
+
45
+ _BACKEND = os.environ.get("RIPRAP_ML_BACKEND", "auto").lower()
46
+ _BASE_URL = os.environ.get("RIPRAP_ML_BASE_URL", "").rstrip("/")
47
+ _API_KEY = os.environ.get("RIPRAP_ML_API_KEY", "")
48
+ _DEFAULT_TIMEOUT = float(os.environ.get("RIPRAP_ML_TIMEOUT_S", "60"))
49
+
50
+
51
+ class RemoteUnreachable(RuntimeError):
52
+ """Raised when the remote inference service is unconfigured, down,
53
+ times out, or returns 5xx. Callers catch this to fall through to a
54
+ local model load. 4xx errors propagate as the generic exception so
55
+ a caller bug doesn't get masked by a "fallback to local" path."""
56
+
57
+
58
+ def remote_enabled() -> bool:
59
+ """True iff the router is configured to attempt remote calls.
60
+ Returns False under explicit `local` mode or when the base URL is
61
+ empty (the auto-default with no env config)."""
62
+ if _BACKEND == "local":
63
+ return False
64
+ if not _BASE_URL:
65
+ return False
66
+ return True
67
+
68
+
69
+ def _client(timeout: float | None = None) -> httpx.Client:
70
+ headers = {"User-Agent": "riprap-app/0.4.5"}
71
+ if _API_KEY:
72
+ headers["Authorization"] = f"Bearer {_API_KEY}"
73
+ return httpx.Client(
74
+ base_url=_BASE_URL,
75
+ headers=headers,
76
+ timeout=timeout if timeout is not None else _DEFAULT_TIMEOUT,
77
+ )
78
+
79
+
80
+ def _post(path: str, payload: dict[str, Any], timeout: float | None = None) -> dict:
81
+ """POST {payload} as JSON to the remote service's `path`. Returns the
82
+ parsed JSON body. Raises RemoteUnreachable on transport errors;
83
+ raises HTTPStatusError on 4xx so caller bugs surface."""
84
+ if not remote_enabled():
85
+ raise RemoteUnreachable("remote ML backend not configured "
86
+ "(RIPRAP_ML_BASE_URL empty or BACKEND=local)")
87
+ t0 = time.monotonic()
88
+ try:
89
+ with _client(timeout) as c:
90
+ r = c.post(path, json=payload)
91
+ except (httpx.ConnectError, httpx.ReadError, httpx.WriteError,
92
+ httpx.TimeoutException, httpx.RemoteProtocolError) as e:
93
+ raise RemoteUnreachable(f"{type(e).__name__}: {e}") from e
94
+ if r.status_code >= 500:
95
+ raise RemoteUnreachable(f"HTTP {r.status_code} from {path}: {r.text[:200]}")
96
+ r.raise_for_status()
97
+ duration_s = time.monotonic() - t0
98
+ # Hardware: msradam/riprap-vllm runs on NVIDIA L4. Operators can
99
+ # override via RIPRAP_HARDWARE_LABEL. The proxy reports per-call
100
+ # GPU energy off NVML in the X-GPU-Energy-J / X-GPU-Power-W headers
101
+ # β€” read those for a real measurement instead of the data-sheet
102
+ # estimate when present.
103
+ override = (os.environ.get("RIPRAP_HARDWARE_LABEL") or "").lower()
104
+ if "mi300x" in override or "amd" in override:
105
+ hw = "amd_mi300x"
106
+ elif "t4" in override:
107
+ hw = "nvidia_t4"
108
+ else:
109
+ hw = "nvidia_l4"
110
+ joules_real, power_w_real = _parse_gpu_headers(r.headers)
111
+ emissions.active().record_ml(
112
+ endpoint=path,
113
+ backend="riprap-models",
114
+ hardware=hw,
115
+ duration_s=duration_s,
116
+ joules_real=joules_real,
117
+ power_w_real=power_w_real,
118
+ )
119
+ return r.json()
120
+
121
+
122
+ def _parse_gpu_headers(headers) -> tuple[float | None, float | None]:
123
+ """Pull (joules, watts) from X-GPU-Energy-J / X-GPU-Power-W if the
124
+ proxy attached them. Returns (None, None) if the headers are absent
125
+ (older proxy build, NVML init failed, or the call streamed)."""
126
+ def _f(name: str) -> float | None:
127
+ v = headers.get(name)
128
+ if v is None or v == "":
129
+ return None
130
+ try:
131
+ return float(v)
132
+ except ValueError:
133
+ return None
134
+ return _f("x-gpu-energy-j"), _f("x-gpu-power-w")
135
+
136
+
137
+ def _serialize_array(arr) -> str:
138
+ """numpy/torch tensor β†’ base64-encoded float32 raw bytes for transport.
139
+ Each remote handler decodes to (shape, dtype=float32) and reconstructs.
140
+ Reasonable round-trip for chips up to a few MB; large rasters should
141
+ use compressed numpy-savez instead β€” TODO when a model needs > 8 MB."""
142
+ import numpy as np
143
+ np_arr = arr if isinstance(arr, np.ndarray) else _to_numpy(arr)
144
+ np_arr = np_arr.astype("float32", copy=False)
145
+ return base64.b64encode(np_arr.tobytes()).decode("ascii")
146
+
147
+
148
+ def _to_numpy(t):
149
+ """Best-effort tensor β†’ numpy. Accepts torch.Tensor or numpy already."""
150
+ try:
151
+ import torch
152
+ if isinstance(t, torch.Tensor):
153
+ return t.detach().cpu().numpy()
154
+ except ImportError:
155
+ pass
156
+ import numpy as np
157
+ return np.asarray(t)
158
+
159
+
160
+ def _deserialize_array(b64: str, shape: list[int]):
161
+ """Inverse of _serialize_array β€” bytes β†’ numpy float32 with given shape."""
162
+ import numpy as np
163
+ raw = base64.b64decode(b64)
164
+ return np.frombuffer(raw, dtype="float32").reshape(shape)
165
+
166
+
167
+ # ---- Public router entry points -------------------------------------------
168
+
169
+ def healthcheck(timeout: float = 3.0) -> bool:
170
+ """Quick reachability probe. True if the service responds 200 to GET
171
+ /healthz within `timeout` seconds. Used by /api/backend so the UI can
172
+ show whether the remote ML backend is currently live."""
173
+ if not remote_enabled():
174
+ return False
175
+ try:
176
+ with _client(timeout) as c:
177
+ r = c.get("/healthz")
178
+ return r.status_code == 200
179
+ except Exception:
180
+ return False
181
+
182
+
183
+ def backend_info() -> dict[str, Any]:
184
+ """Snapshot for /api/backend β€” what the UI should advertise."""
185
+ return {
186
+ "backend": _BACKEND,
187
+ "base_url": _BASE_URL or None,
188
+ "remote_enabled": remote_enabled(),
189
+ "reachable": healthcheck() if remote_enabled() else False,
190
+ }
191
+
192
+
193
+ def prithvi_pluvial(s2_chip, *, scene_id: str | None = None,
194
+ scene_datetime: str | None = None,
195
+ cloud_cover: float | None = None,
196
+ timeout: float | None = None) -> dict[str, Any]:
197
+ """Remote forward pass through Prithvi-NYC-Pluvial v2.
198
+ Input: 6-band Sentinel-2 chip (numpy or torch, shape [6, H, W]).
199
+ Output: { ok, pct_water_within_500m, pct_water_full, scene_id, ... }.
200
+ Raises RemoteUnreachable if the service is down."""
201
+ arr = _to_numpy(s2_chip)
202
+ return _post("/v1/prithvi-pluvial", {
203
+ "s2": _serialize_array(arr),
204
+ "shape": list(arr.shape),
205
+ "scene_id": scene_id,
206
+ "scene_datetime": scene_datetime,
207
+ "cloud_cover": cloud_cover,
208
+ }, timeout=timeout)
209
+
210
+
211
+ def terramind(adapter: str, s2l2a=None, s1rtc=None, dem=None, *,
212
+ timeout: float | None = None) -> dict[str, Any]:
213
+ """Remote forward through TerraMind-NYC-Adapters (LULC or Buildings)
214
+ or the v1 base generative path (synthesis). `adapter` is one of:
215
+ lulc, buildings, synthesis. Each modality is a numpy array, torch
216
+ tensor, or None β€” `synthesis` only needs DEM; the LoRA adapters
217
+ need at minimum S2L2A."""
218
+ payload: dict[str, Any] = {"adapter": adapter}
219
+ if s2l2a is not None:
220
+ s2_np = _to_numpy(s2l2a)
221
+ payload["s2"] = _serialize_array(s2_np)
222
+ payload["s2_shape"] = list(s2_np.shape)
223
+ if s1rtc is not None:
224
+ s1_np = _to_numpy(s1rtc)
225
+ payload["s1"] = _serialize_array(s1_np)
226
+ payload["s1_shape"] = list(s1_np.shape)
227
+ if dem is not None:
228
+ dem_np = _to_numpy(dem)
229
+ payload["dem"] = _serialize_array(dem_np)
230
+ payload["dem_shape"] = list(dem_np.shape)
231
+ return _post("/v1/terramind", payload, timeout=timeout)
232
+
233
+
234
+ def ttm_forecast(model: str, history: Iterable[float], *,
235
+ context_length: int, prediction_length: int,
236
+ cadence: str = "h",
237
+ timeout: float | None = None) -> dict[str, Any]:
238
+ """Remote Granite TTM r2 forecast.
239
+ `model` is one of: zero_shot_battery, fine_tune_battery, weekly_311,
240
+ floodnet_recurrence β€” the service decides which checkpoint to use.
241
+ `history` is a 1-D iterable of floats (the time series); `cadence`
242
+ is for the service's labelling (h / d / w / 6m). Output shape is
243
+ `{ ok, forecast: [...], peak_index, peak_value }`."""
244
+ series = list(map(float, history))
245
+ return _post("/v1/ttm-forecast", {
246
+ "model": model,
247
+ "history": series,
248
+ "context_length": context_length,
249
+ "prediction_length": prediction_length,
250
+ "cadence": cadence,
251
+ }, timeout=timeout)
252
+
253
+
254
+ def granite_embed(texts: list[str], *,
255
+ timeout: float | None = None) -> dict[str, Any]:
256
+ """Remote Granite Embedding 278M batch encode.
257
+ Output: { ok, vectors: [[float, ...], ...] }. Vector dimension fixed
258
+ at 768 (granite-embedding-278m-multilingual)."""
259
+ return _post("/v1/granite-embed", {"texts": list(texts)}, timeout=timeout)
260
+
261
+
262
+ def gliner_extract(text: str, labels: list[str], *,
263
+ timeout: float | None = None) -> dict[str, Any]:
264
+ """Remote GLiNER typed-entity extraction.
265
+ Output: { ok, entities: [{label, text, start, end, score}, ...] }."""
266
+ return _post("/v1/gliner-extract", {
267
+ "text": text, "labels": list(labels),
268
+ }, timeout=timeout)
app/intents/__init__.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ """Per-intent execution modules. Each intent knows how to take a planner
2
+ Plan and run only the specialists relevant to it, returning a
3
+ reconciler-ready set of documents and a paragraph."""