KSvend Claude Happy commited on
Commit ·
bf99ddb
1
Parent(s): b7f7fb5
feat: implement NdviIndicator.harvest() with graceful degradation
Browse filesGenerated with [Claude Code](https://claude.ai/code)
via [Happy](https://happy.engineering)
Co-Authored-By: Claude <noreply@anthropic.com>
Co-Authored-By: Happy <yesreply@happy.engineering>
- app/indicators/ndvi.py +126 -0
- tests/test_indicator_ndvi.py +112 -0
app/indicators/ndvi.py
CHANGED
|
@@ -84,6 +84,119 @@ class NdviIndicator(BaseIndicator):
|
|
| 84 |
submit_as_batch(conn, true_color_cube, f"ndvi-truecolor-{aoi.name}"),
|
| 85 |
]
|
| 86 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
async def process(
|
| 88 |
self, aoi: AOI, time_range: TimeRange, season_months: list[int] | None = None
|
| 89 |
) -> IndicatorResult:
|
|
@@ -299,6 +412,19 @@ class NdviIndicator(BaseIndicator):
|
|
| 299 |
"label": "NDVI",
|
| 300 |
}
|
| 301 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 302 |
def _fallback(self, aoi: AOI, time_range: TimeRange) -> IndicatorResult:
|
| 303 |
"""Return a placeholder result when openEO processing fails."""
|
| 304 |
rng = np.random.default_rng(7)
|
|
|
|
| 84 |
submit_as_batch(conn, true_color_cube, f"ndvi-truecolor-{aoi.name}"),
|
| 85 |
]
|
| 86 |
|
| 87 |
+
async def harvest(
|
| 88 |
+
self, aoi: AOI, time_range: TimeRange, season_months: list[int] | None = None,
|
| 89 |
+
batch_jobs: list | None = None,
|
| 90 |
+
) -> IndicatorResult:
|
| 91 |
+
"""Download completed batch job results and compute NDVI statistics."""
|
| 92 |
+
current_job, baseline_job, true_color_job = batch_jobs
|
| 93 |
+
|
| 94 |
+
results_dir = tempfile.mkdtemp(prefix="aperture_ndvi_batch_")
|
| 95 |
+
|
| 96 |
+
# Download current NDVI — required
|
| 97 |
+
try:
|
| 98 |
+
current_dir = os.path.join(results_dir, "current")
|
| 99 |
+
paths = current_job.download_results(current_dir)
|
| 100 |
+
current_path = self._find_tif(paths, current_dir)
|
| 101 |
+
except Exception as exc:
|
| 102 |
+
logger.warning("NDVI current batch download failed: %s", exc)
|
| 103 |
+
return self._fallback(aoi, time_range)
|
| 104 |
+
|
| 105 |
+
# Download baseline — optional (degrades gracefully)
|
| 106 |
+
baseline_path = None
|
| 107 |
+
try:
|
| 108 |
+
baseline_dir = os.path.join(results_dir, "baseline")
|
| 109 |
+
paths = baseline_job.download_results(baseline_dir)
|
| 110 |
+
baseline_path = self._find_tif(paths, baseline_dir)
|
| 111 |
+
except Exception as exc:
|
| 112 |
+
logger.warning("NDVI baseline batch download failed, degrading: %s", exc)
|
| 113 |
+
|
| 114 |
+
# Download true-color — optional
|
| 115 |
+
true_color_path = None
|
| 116 |
+
try:
|
| 117 |
+
tc_dir = os.path.join(results_dir, "truecolor")
|
| 118 |
+
paths = true_color_job.download_results(tc_dir)
|
| 119 |
+
true_color_path = self._find_tif(paths, tc_dir)
|
| 120 |
+
except Exception as exc:
|
| 121 |
+
logger.warning("NDVI true-color batch download failed: %s", exc)
|
| 122 |
+
|
| 123 |
+
# Compute statistics
|
| 124 |
+
current_stats = self._compute_stats(current_path)
|
| 125 |
+
current_mean = current_stats["overall_mean"]
|
| 126 |
+
|
| 127 |
+
if baseline_path:
|
| 128 |
+
baseline_stats = self._compute_stats(baseline_path)
|
| 129 |
+
baseline_mean = baseline_stats["overall_mean"]
|
| 130 |
+
change = current_mean - baseline_mean
|
| 131 |
+
confidence = (
|
| 132 |
+
ConfidenceLevel.HIGH if current_stats["valid_months"] >= 6
|
| 133 |
+
else ConfidenceLevel.MODERATE if current_stats["valid_months"] >= 3
|
| 134 |
+
else ConfidenceLevel.LOW
|
| 135 |
+
)
|
| 136 |
+
chart_data = self._build_chart_data(
|
| 137 |
+
current_stats["monthly_means"],
|
| 138 |
+
baseline_stats["monthly_means"],
|
| 139 |
+
time_range,
|
| 140 |
+
)
|
| 141 |
+
else:
|
| 142 |
+
baseline_mean = current_mean
|
| 143 |
+
change = 0.0
|
| 144 |
+
confidence = ConfidenceLevel.LOW
|
| 145 |
+
chart_data = {
|
| 146 |
+
"dates": [f"{time_range.end.year}-{m+1:02d}" for m in range(len(current_stats["monthly_means"]))],
|
| 147 |
+
"values": [round(v, 3) for v in current_stats["monthly_means"]],
|
| 148 |
+
"label": "NDVI",
|
| 149 |
+
}
|
| 150 |
+
|
| 151 |
+
status = self._classify(change)
|
| 152 |
+
trend = self._compute_trend(change) if baseline_path else TrendDirection.STABLE
|
| 153 |
+
|
| 154 |
+
if abs(change) <= 0.05:
|
| 155 |
+
headline = f"Vegetation stable (NDVI {current_mean:.2f}, \u0394{change:+.2f} vs baseline)"
|
| 156 |
+
elif change > 0:
|
| 157 |
+
headline = f"Vegetation greening (NDVI +{change:.2f} vs baseline)"
|
| 158 |
+
else:
|
| 159 |
+
headline = f"Vegetation decline (NDVI {change:.2f} vs baseline)"
|
| 160 |
+
|
| 161 |
+
self._spatial_data = SpatialData(
|
| 162 |
+
map_type="raster", label="NDVI", colormap="RdYlGn",
|
| 163 |
+
vmin=-0.2, vmax=0.9,
|
| 164 |
+
)
|
| 165 |
+
self._indicator_raster_path = current_path
|
| 166 |
+
self._true_color_path = true_color_path
|
| 167 |
+
self._ndvi_peak_band = current_stats["peak_month_band"]
|
| 168 |
+
self._render_band = current_stats["peak_month_band"]
|
| 169 |
+
|
| 170 |
+
return IndicatorResult(
|
| 171 |
+
indicator_id=self.id,
|
| 172 |
+
headline=headline,
|
| 173 |
+
status=status,
|
| 174 |
+
trend=trend,
|
| 175 |
+
confidence=confidence,
|
| 176 |
+
map_layer_path=current_path,
|
| 177 |
+
chart_data=chart_data,
|
| 178 |
+
data_source="satellite",
|
| 179 |
+
summary=(
|
| 180 |
+
f"Mean NDVI is {current_mean:.3f} compared to a {BASELINE_YEARS}-year "
|
| 181 |
+
f"baseline of {baseline_mean:.3f} (\u0394{change:+.3f}). "
|
| 182 |
+
f"Pixel-level analysis at {RESOLUTION_M}m resolution from "
|
| 183 |
+
f"{current_stats['valid_months']} monthly composites."
|
| 184 |
+
),
|
| 185 |
+
methodology=(
|
| 186 |
+
f"Sentinel-2 L2A pixel-level NDVI = (B08 \u2212 B04) / (B08 + B04). "
|
| 187 |
+
f"Cloud-masked using SCL band (classes 4, 5, 6 retained). "
|
| 188 |
+
f"Monthly median composites at {RESOLUTION_M}m resolution. "
|
| 189 |
+
f"Baseline: {BASELINE_YEARS}-year monthly medians. "
|
| 190 |
+
f"Processed server-side via CDSE openEO batch jobs."
|
| 191 |
+
),
|
| 192 |
+
limitations=[
|
| 193 |
+
f"Resampled to {RESOLUTION_M}m \u2014 sub-field variability not captured at this resolution.",
|
| 194 |
+
"Cloud cover reduces observation count in rainy seasons.",
|
| 195 |
+
"NDVI does not distinguish crop from natural vegetation.",
|
| 196 |
+
"Seasonal variation may mask long-term trends if analysis windows differ.",
|
| 197 |
+
] + (["Baseline unavailable \u2014 change and trend not computed."] if not baseline_path else []),
|
| 198 |
+
)
|
| 199 |
+
|
| 200 |
async def process(
|
| 201 |
self, aoi: AOI, time_range: TimeRange, season_months: list[int] | None = None
|
| 202 |
) -> IndicatorResult:
|
|
|
|
| 412 |
"label": "NDVI",
|
| 413 |
}
|
| 414 |
|
| 415 |
+
@staticmethod
|
| 416 |
+
def _find_tif(download_paths: dict, fallback_dir: str) -> str:
|
| 417 |
+
"""Find the GeoTIFF file from batch job download results."""
|
| 418 |
+
if download_paths:
|
| 419 |
+
for p in download_paths:
|
| 420 |
+
if str(p).endswith(".tif") or str(p).endswith(".tiff"):
|
| 421 |
+
return str(p)
|
| 422 |
+
# Fallback: look for any .tif in the directory
|
| 423 |
+
for f in os.listdir(fallback_dir):
|
| 424 |
+
if f.endswith(".tif") or f.endswith(".tiff"):
|
| 425 |
+
return os.path.join(fallback_dir, f)
|
| 426 |
+
raise FileNotFoundError(f"No GeoTIFF found in {fallback_dir}")
|
| 427 |
+
|
| 428 |
def _fallback(self, aoi: AOI, time_range: TimeRange) -> IndicatorResult:
|
| 429 |
"""Return a placeholder result when openEO processing fails."""
|
| 430 |
rng = np.random.default_rng(7)
|
tests/test_indicator_ndvi.py
CHANGED
|
@@ -151,3 +151,115 @@ async def test_ndvi_submit_batch_creates_three_jobs(test_aoi, test_time_range):
|
|
| 151 |
assert len(jobs) == 3
|
| 152 |
assert mock_ndvi_graph.call_count == 2 # current + baseline
|
| 153 |
assert mock_tc_graph.call_count == 1 # true-color
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 151 |
assert len(jobs) == 3
|
| 152 |
assert mock_ndvi_graph.call_count == 2 # current + baseline
|
| 153 |
assert mock_tc_graph.call_count == 1 # true-color
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
@pytest.mark.asyncio
|
| 157 |
+
async def test_ndvi_harvest_computes_result_from_batch_jobs(test_aoi, test_time_range):
|
| 158 |
+
"""harvest() downloads batch results and returns IndicatorResult."""
|
| 159 |
+
from app.indicators.ndvi import NdviIndicator
|
| 160 |
+
|
| 161 |
+
indicator = NdviIndicator()
|
| 162 |
+
|
| 163 |
+
with tempfile.TemporaryDirectory() as tmpdir:
|
| 164 |
+
ndvi_path = os.path.join(tmpdir, "ndvi.tif")
|
| 165 |
+
rgb_path = os.path.join(tmpdir, "rgb.tif")
|
| 166 |
+
_mock_ndvi_tif(ndvi_path)
|
| 167 |
+
_mock_true_color_tif(rgb_path)
|
| 168 |
+
|
| 169 |
+
def make_mock_job(src_path):
|
| 170 |
+
job = MagicMock()
|
| 171 |
+
job.job_id = "j-test"
|
| 172 |
+
|
| 173 |
+
def fake_download_results(target):
|
| 174 |
+
import shutil
|
| 175 |
+
os.makedirs(target, exist_ok=True)
|
| 176 |
+
dest = os.path.join(target, "result.tif")
|
| 177 |
+
shutil.copy(src_path, dest)
|
| 178 |
+
from pathlib import Path
|
| 179 |
+
return {Path(dest): {"type": "image/tiff"}}
|
| 180 |
+
job.download_results.side_effect = fake_download_results
|
| 181 |
+
job.status.return_value = "finished"
|
| 182 |
+
return job
|
| 183 |
+
|
| 184 |
+
current_job = make_mock_job(ndvi_path)
|
| 185 |
+
baseline_job = make_mock_job(ndvi_path)
|
| 186 |
+
true_color_job = make_mock_job(rgb_path)
|
| 187 |
+
|
| 188 |
+
result = await indicator.harvest(
|
| 189 |
+
test_aoi, test_time_range,
|
| 190 |
+
batch_jobs=[current_job, baseline_job, true_color_job],
|
| 191 |
+
)
|
| 192 |
+
|
| 193 |
+
assert result.indicator_id == "ndvi"
|
| 194 |
+
assert result.data_source == "satellite"
|
| 195 |
+
assert result.status in (StatusLevel.GREEN, StatusLevel.AMBER, StatusLevel.RED)
|
| 196 |
+
assert result.confidence in (ConfidenceLevel.HIGH, ConfidenceLevel.MODERATE, ConfidenceLevel.LOW)
|
| 197 |
+
assert len(result.chart_data.get("dates", [])) > 0
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
@pytest.mark.asyncio
|
| 201 |
+
async def test_ndvi_harvest_degrades_when_baseline_fails(test_aoi, test_time_range):
|
| 202 |
+
"""harvest() returns partial result when baseline job failed."""
|
| 203 |
+
from app.indicators.ndvi import NdviIndicator
|
| 204 |
+
|
| 205 |
+
indicator = NdviIndicator()
|
| 206 |
+
|
| 207 |
+
with tempfile.TemporaryDirectory() as tmpdir:
|
| 208 |
+
ndvi_path = os.path.join(tmpdir, "ndvi.tif")
|
| 209 |
+
rgb_path = os.path.join(tmpdir, "rgb.tif")
|
| 210 |
+
_mock_ndvi_tif(ndvi_path)
|
| 211 |
+
_mock_true_color_tif(rgb_path)
|
| 212 |
+
|
| 213 |
+
def make_mock_job(src_path, status="finished"):
|
| 214 |
+
job = MagicMock()
|
| 215 |
+
job.job_id = "j-test"
|
| 216 |
+
job.status.return_value = status
|
| 217 |
+
|
| 218 |
+
def fake_download_results(target):
|
| 219 |
+
if status == "error":
|
| 220 |
+
raise Exception("Batch job failed on CDSE")
|
| 221 |
+
os.makedirs(target, exist_ok=True)
|
| 222 |
+
dest = os.path.join(target, "result.tif")
|
| 223 |
+
import shutil
|
| 224 |
+
shutil.copy(src_path, dest)
|
| 225 |
+
from pathlib import Path
|
| 226 |
+
return {Path(dest): {"type": "image/tiff"}}
|
| 227 |
+
job.download_results.side_effect = fake_download_results
|
| 228 |
+
return job
|
| 229 |
+
|
| 230 |
+
current_job = make_mock_job(ndvi_path)
|
| 231 |
+
baseline_job = make_mock_job(ndvi_path, status="error")
|
| 232 |
+
true_color_job = make_mock_job(rgb_path)
|
| 233 |
+
|
| 234 |
+
result = await indicator.harvest(
|
| 235 |
+
test_aoi, test_time_range,
|
| 236 |
+
batch_jobs=[current_job, baseline_job, true_color_job],
|
| 237 |
+
)
|
| 238 |
+
|
| 239 |
+
assert result.indicator_id == "ndvi"
|
| 240 |
+
assert result.data_source == "satellite"
|
| 241 |
+
assert result.confidence == ConfidenceLevel.LOW
|
| 242 |
+
assert result.trend == TrendDirection.STABLE
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
@pytest.mark.asyncio
|
| 246 |
+
async def test_ndvi_harvest_falls_back_when_current_fails(test_aoi, test_time_range):
|
| 247 |
+
"""harvest() returns placeholder when current NDVI job failed."""
|
| 248 |
+
from app.indicators.ndvi import NdviIndicator
|
| 249 |
+
|
| 250 |
+
indicator = NdviIndicator()
|
| 251 |
+
|
| 252 |
+
current_job = MagicMock()
|
| 253 |
+
current_job.status.return_value = "error"
|
| 254 |
+
current_job.download_results.side_effect = Exception("failed")
|
| 255 |
+
baseline_job = MagicMock()
|
| 256 |
+
baseline_job.status.return_value = "finished"
|
| 257 |
+
true_color_job = MagicMock()
|
| 258 |
+
true_color_job.status.return_value = "finished"
|
| 259 |
+
|
| 260 |
+
result = await indicator.harvest(
|
| 261 |
+
test_aoi, test_time_range,
|
| 262 |
+
batch_jobs=[current_job, baseline_job, true_color_job],
|
| 263 |
+
)
|
| 264 |
+
|
| 265 |
+
assert result.data_source == "placeholder"
|