File size: 3,650 Bytes
d7c810d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
"""Sanity checks on the built cube. Prints a report and writes data/cube/qa_report.json.

Checks are descriptive (no thresholds tuned to make things pass); the report is meant to be read.
"""
from __future__ import annotations

import json
from pathlib import Path

import numpy as np
import pandas as pd

CUBE = Path("/root/trendex-ml/data/cube")
PLACES = {"Son-Kul pastures (N shore)": (41.90, 75.15), "Naryn city": (41.43, 75.99),
          "At-Bashy valley": (41.17, 75.80), "Suusamyr valley": (42.15, 74.00)}


def nearest_cell(cells: pd.DataFrame, lat: float, lon: float) -> int:
    d = (cells["lat"] - lat) ** 2 + ((cells["lon"] - lon) * np.cos(np.deg2rad(lat))) ** 2
    return int(cells.loc[d.idxmin(), "cell_id"])


def main() -> None:
    cells = pd.read_parquet(CUBE / "cells.parquet")
    rep: dict = {"cells": len(cells), "in_oblast": int(cells["in_oblast"].sum()),
                 "boundary_conflict": int(cells["boundary_conflict"].sum()),
                 "by_district": {str(k): int(v) for k, v in cells["district"].value_counts(dropna=False).items()},
                 "mask_reason": {str(k): int(v) for k, v in cells["mask_reason"].value_counts(dropna=False).items()}}
    num = ["elev_mean", "elev_std", "slope_mean", "steep_share", "tpi", "grass_frac", "water_frac", "ice_frac"]
    rep["static_ranges"] = cells[num].describe(percentiles=[0.01, 0.5, 0.99]).round(3).to_dict()
    rep["static_missing_share"] = cells[num + ["lc_type1"]].isna().mean().round(4).to_dict()

    monthly_dir = CUBE / "monthly"
    if monthly_dir.exists():
        m = pd.read_parquet(monthly_dir)
        m["year"] = m["year"].astype(int)
        m = m.merge(cells[["cell_id", "in_oblast", "mask_reason"]], on="cell_id")
        past = m[m["in_oblast"] & m["mask_reason"].isna()]
        rep["monthly_rows"] = len(m)
        rep["years"] = [int(m["year"].min()), int(m["year"].max())]
        rep["ndvi_ok_share_by_month"] = past.groupby("month")["ndvi_ok"].mean().round(3).to_dict()
        cols = ["ndvi", "ndvi_aqua", "lst_day", "lst_night", "snow", "precip", "soil_moist", "swe"]
        rep["monthly_ranges"] = past[cols].describe(percentiles=[0.01, 0.5, 0.99]).round(3).to_dict()
        rep["monthly_missing_share_MaySep"] = past[past["month"].between(5, 9)][cols].isna().mean().round(3).to_dict()
        both = past[past["ndvi_ok"] & past["ndvi_aqua"].notna()]
        rep["terra_aqua_ndvi_corr"] = round(float(both["ndvi"].corr(both["ndvi_aqua"])), 3) if len(both) else None
        summer = past[past["month"].between(6, 8) & past["ndvi_ok"]]
        rep["oblast_summer_ndvi_by_year"] = summer.groupby("year")["ndvi"].mean().round(4).to_dict()
        rep["places"] = {}
        for name, (lat, lon) in PLACES.items():
            cid = nearest_cell(cells, lat, lon)
            s = m[(m["cell_id"] == cid) & m["month"].between(6, 8)]
            row = cells[cells["cell_id"] == cid].iloc[0]
            rep["places"][name] = {
                "cell_id": cid, "district": None if pd.isna(row["district"]) else row["district"],
                "mask_reason": None if pd.isna(row["mask_reason"]) else row["mask_reason"],
                "elev_m": round(float(row["elev_mean"]), 0),
                "summer_ndvi_mean": round(float(s.loc[s["ndvi_ok"], "ndvi"].mean()), 3),
                "summer_precip_mm_mean": round(float(s["precip"].mean()), 1),
                "summer_lst_day_c_mean": round(float(s["lst_day"].mean()), 1)}
    (CUBE / "qa_report.json").write_text(json.dumps(rep, indent=2, default=str, ensure_ascii=False))
    print(json.dumps(rep, indent=1, default=str, ensure_ascii=False))


if __name__ == "__main__":
    main()