File size: 3,650 Bytes
d7c810d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | """Sanity checks on the built cube. Prints a report and writes data/cube/qa_report.json.
Checks are descriptive (no thresholds tuned to make things pass); the report is meant to be read.
"""
from __future__ import annotations
import json
from pathlib import Path
import numpy as np
import pandas as pd
CUBE = Path("/root/trendex-ml/data/cube")
PLACES = {"Son-Kul pastures (N shore)": (41.90, 75.15), "Naryn city": (41.43, 75.99),
"At-Bashy valley": (41.17, 75.80), "Suusamyr valley": (42.15, 74.00)}
def nearest_cell(cells: pd.DataFrame, lat: float, lon: float) -> int:
d = (cells["lat"] - lat) ** 2 + ((cells["lon"] - lon) * np.cos(np.deg2rad(lat))) ** 2
return int(cells.loc[d.idxmin(), "cell_id"])
def main() -> None:
cells = pd.read_parquet(CUBE / "cells.parquet")
rep: dict = {"cells": len(cells), "in_oblast": int(cells["in_oblast"].sum()),
"boundary_conflict": int(cells["boundary_conflict"].sum()),
"by_district": {str(k): int(v) for k, v in cells["district"].value_counts(dropna=False).items()},
"mask_reason": {str(k): int(v) for k, v in cells["mask_reason"].value_counts(dropna=False).items()}}
num = ["elev_mean", "elev_std", "slope_mean", "steep_share", "tpi", "grass_frac", "water_frac", "ice_frac"]
rep["static_ranges"] = cells[num].describe(percentiles=[0.01, 0.5, 0.99]).round(3).to_dict()
rep["static_missing_share"] = cells[num + ["lc_type1"]].isna().mean().round(4).to_dict()
monthly_dir = CUBE / "monthly"
if monthly_dir.exists():
m = pd.read_parquet(monthly_dir)
m["year"] = m["year"].astype(int)
m = m.merge(cells[["cell_id", "in_oblast", "mask_reason"]], on="cell_id")
past = m[m["in_oblast"] & m["mask_reason"].isna()]
rep["monthly_rows"] = len(m)
rep["years"] = [int(m["year"].min()), int(m["year"].max())]
rep["ndvi_ok_share_by_month"] = past.groupby("month")["ndvi_ok"].mean().round(3).to_dict()
cols = ["ndvi", "ndvi_aqua", "lst_day", "lst_night", "snow", "precip", "soil_moist", "swe"]
rep["monthly_ranges"] = past[cols].describe(percentiles=[0.01, 0.5, 0.99]).round(3).to_dict()
rep["monthly_missing_share_MaySep"] = past[past["month"].between(5, 9)][cols].isna().mean().round(3).to_dict()
both = past[past["ndvi_ok"] & past["ndvi_aqua"].notna()]
rep["terra_aqua_ndvi_corr"] = round(float(both["ndvi"].corr(both["ndvi_aqua"])), 3) if len(both) else None
summer = past[past["month"].between(6, 8) & past["ndvi_ok"]]
rep["oblast_summer_ndvi_by_year"] = summer.groupby("year")["ndvi"].mean().round(4).to_dict()
rep["places"] = {}
for name, (lat, lon) in PLACES.items():
cid = nearest_cell(cells, lat, lon)
s = m[(m["cell_id"] == cid) & m["month"].between(6, 8)]
row = cells[cells["cell_id"] == cid].iloc[0]
rep["places"][name] = {
"cell_id": cid, "district": None if pd.isna(row["district"]) else row["district"],
"mask_reason": None if pd.isna(row["mask_reason"]) else row["mask_reason"],
"elev_m": round(float(row["elev_mean"]), 0),
"summer_ndvi_mean": round(float(s.loc[s["ndvi_ok"], "ndvi"].mean()), 3),
"summer_precip_mm_mean": round(float(s["precip"].mean()), 1),
"summer_lst_day_c_mean": round(float(s["lst_day"].mean()), 1)}
(CUBE / "qa_report.json").write_text(json.dumps(rep, indent=2, default=str, ensure_ascii=False))
print(json.dumps(rep, indent=1, default=str, ensure_ascii=False))
if __name__ == "__main__":
main()
|