"""Sanity checks on the built cube. Prints a report and writes data/cube/qa_report.json. Checks are descriptive (no thresholds tuned to make things pass); the report is meant to be read. """ from __future__ import annotations import json from pathlib import Path import numpy as np import pandas as pd CUBE = Path("/root/trendex-ml/data/cube") PLACES = {"Son-Kul pastures (N shore)": (41.90, 75.15), "Naryn city": (41.43, 75.99), "At-Bashy valley": (41.17, 75.80), "Suusamyr valley": (42.15, 74.00)} def nearest_cell(cells: pd.DataFrame, lat: float, lon: float) -> int: d = (cells["lat"] - lat) ** 2 + ((cells["lon"] - lon) * np.cos(np.deg2rad(lat))) ** 2 return int(cells.loc[d.idxmin(), "cell_id"]) def main() -> None: cells = pd.read_parquet(CUBE / "cells.parquet") rep: dict = {"cells": len(cells), "in_oblast": int(cells["in_oblast"].sum()), "boundary_conflict": int(cells["boundary_conflict"].sum()), "by_district": {str(k): int(v) for k, v in cells["district"].value_counts(dropna=False).items()}, "mask_reason": {str(k): int(v) for k, v in cells["mask_reason"].value_counts(dropna=False).items()}} num = ["elev_mean", "elev_std", "slope_mean", "steep_share", "tpi", "grass_frac", "water_frac", "ice_frac"] rep["static_ranges"] = cells[num].describe(percentiles=[0.01, 0.5, 0.99]).round(3).to_dict() rep["static_missing_share"] = cells[num + ["lc_type1"]].isna().mean().round(4).to_dict() monthly_dir = CUBE / "monthly" if monthly_dir.exists(): m = pd.read_parquet(monthly_dir) m["year"] = m["year"].astype(int) m = m.merge(cells[["cell_id", "in_oblast", "mask_reason"]], on="cell_id") past = m[m["in_oblast"] & m["mask_reason"].isna()] rep["monthly_rows"] = len(m) rep["years"] = [int(m["year"].min()), int(m["year"].max())] rep["ndvi_ok_share_by_month"] = past.groupby("month")["ndvi_ok"].mean().round(3).to_dict() cols = ["ndvi", "ndvi_aqua", "lst_day", "lst_night", "snow", "precip", "soil_moist", "swe"] rep["monthly_ranges"] = past[cols].describe(percentiles=[0.01, 0.5, 0.99]).round(3).to_dict() rep["monthly_missing_share_MaySep"] = past[past["month"].between(5, 9)][cols].isna().mean().round(3).to_dict() both = past[past["ndvi_ok"] & past["ndvi_aqua"].notna()] rep["terra_aqua_ndvi_corr"] = round(float(both["ndvi"].corr(both["ndvi_aqua"])), 3) if len(both) else None summer = past[past["month"].between(6, 8) & past["ndvi_ok"]] rep["oblast_summer_ndvi_by_year"] = summer.groupby("year")["ndvi"].mean().round(4).to_dict() rep["places"] = {} for name, (lat, lon) in PLACES.items(): cid = nearest_cell(cells, lat, lon) s = m[(m["cell_id"] == cid) & m["month"].between(6, 8)] row = cells[cells["cell_id"] == cid].iloc[0] rep["places"][name] = { "cell_id": cid, "district": None if pd.isna(row["district"]) else row["district"], "mask_reason": None if pd.isna(row["mask_reason"]) else row["mask_reason"], "elev_m": round(float(row["elev_mean"]), 0), "summer_ndvi_mean": round(float(s.loc[s["ndvi_ok"], "ndvi"].mean()), 3), "summer_precip_mm_mean": round(float(s["precip"].mean()), 1), "summer_lst_day_c_mean": round(float(s["lst_day"].mean()), 1)} (CUBE / "qa_report.json").write_text(json.dumps(rep, indent=2, default=str, ensure_ascii=False)) print(json.dumps(rep, indent=1, default=str, ensure_ascii=False)) if __name__ == "__main__": main()