File size: 6,156 Bytes
7e6579d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 | """Trend statistics, shared by the formula (F3) and the ML session's residual trends.
One implementation, copied (not forked) into the ML repo.
- trend_1d: Sen slope + Mann-Kendall (Hamed-Rao autocorrelation correction by default) + 95% CI (Gilbert 1987).
- trend_many: trend_1d over rows of a 2-D array (joblib).
- mk_vectorized / sen_vectorized: fast numpy versions (original MK, no autocorrelation correction),
used for placebo permutations and sensitivity loops.
- bh_fdr: Benjamini-Hochberg q-values.
- classify: growth / decline / unreliable / insufficient.
"""
from __future__ import annotations
import math
import warnings
import numpy as np
import pymannkendall as pmk
from scipy.stats import norm
CLASSES = ("growth", "decline", "unreliable", "insufficient")
def _sen_ci(y: np.ndarray, t: np.ndarray, alpha: float = 0.05) -> tuple[float, float, float]:
"""Sen slope and its (1-alpha) CI by the rank method (Gilbert 1987). y, t without NaN."""
n = len(y)
i, j = np.triu_indices(n, k=1)
dt = t[j] - t[i]
slopes = np.sort((y[j] - y[i]) / dt)
m = len(slopes)
med = float(np.median(slopes))
# variance of S without tie correction (ties are rare for float NDVI means)
var_s = n * (n - 1) * (2 * n + 5) / 18.0
c = norm.ppf(1 - alpha / 2) * math.sqrt(var_s)
lo_idx = int(math.floor((m - c) / 2.0))
hi_idx = int(math.ceil((m + c) / 2.0))
lo_idx = min(max(lo_idx, 0), m - 1)
hi_idx = min(max(hi_idx, 0), m - 1)
return med, float(slopes[lo_idx]), float(slopes[hi_idx])
def trend_1d(y, years, method: str = "hamed_rao", alpha: float = 0.05, min_n: int = 8) -> dict:
"""Trend of one series. NaNs are dropped. Returns slope per time unit (per year)."""
y = np.asarray(y, dtype=float)
t = np.asarray(years, dtype=float)
ok = np.isfinite(y)
y, t = y[ok], t[ok]
n = int(len(y))
out = {"slope": np.nan, "intercept": np.nan, "ci_lo": np.nan, "ci_hi": np.nan,
"p": np.nan, "z": np.nan, "tau": np.nan, "n": n, "method": method}
if n < min_n or np.nanstd(y) == 0:
if n >= min_n:
out.update(slope=0.0, intercept=float(np.median(y)), ci_lo=0.0, ci_hi=0.0, p=1.0, z=0.0, tau=0.0)
return out
slope, lo, hi = _sen_ci(y, t, alpha)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
if method == "hamed_rao":
r = pmk.hamed_rao_modification_test(y)
else:
r = pmk.original_test(y)
p = float(r.p)
if not np.isfinite(p): # Hamed-Rao can fail on degenerate series; fall back to original MK
r = pmk.original_test(y)
p = float(r.p)
intercept = float(np.median(y - slope * t))
out.update(slope=slope, intercept=intercept, ci_lo=lo, ci_hi=hi, p=p, z=float(r.z), tau=float(r.Tau))
return out
def _trend_rows(Y, years, method, alpha, min_n):
return [trend_1d(row, years, method, alpha, min_n) for row in Y]
def trend_many(Y, years, method: str = "hamed_rao", alpha: float = 0.05, min_n: int = 8,
n_jobs: int = -1, chunk: int = 2000):
"""trend_1d for every row of Y (n_series, n_years). Returns a dict of 1-D arrays."""
import pandas as pd
from joblib import Parallel, delayed
Y = np.asarray(Y, dtype=float)
chunks = [Y[i:i + chunk] for i in range(0, len(Y), chunk)]
if len(chunks) <= 1 or n_jobs == 1:
rows = _trend_rows(Y, years, method, alpha, min_n)
else:
parts = Parallel(n_jobs=n_jobs)(delayed(_trend_rows)(c, years, method, alpha, min_n) for c in chunks)
rows = [r for p in parts for r in p]
df = pd.DataFrame(rows)
return {k: df[k].to_numpy() for k in df.columns}
def sen_vectorized(Y, years) -> np.ndarray:
"""Sen slope for each row (NaN-aware)."""
Y = np.asarray(Y, dtype=np.float32)
t = np.asarray(years, dtype=np.float32)
i, j = np.triu_indices(len(t), k=1)
s = (Y[:, j] - Y[:, i]) / (t[j] - t[i])
with warnings.catch_warnings():
warnings.simplefilter("ignore")
return np.nanmedian(s, axis=1)
def mk_vectorized(Y, years=None):
"""Original Mann-Kendall for each row (NaN-aware, no tie/autocorrelation correction).
Returns (S, var_s, z, p)."""
Y = np.asarray(Y, dtype=np.float32)
n_t = Y.shape[1]
i, j = np.triu_indices(n_t, k=1)
d = Y[:, j] - Y[:, i]
valid = np.isfinite(d)
S = np.where(valid, np.sign(d), 0).sum(axis=1).astype(float)
n = np.isfinite(Y).sum(axis=1).astype(float)
var_s = n * (n - 1) * (2 * n + 5) / 18.0
with np.errstate(invalid="ignore", divide="ignore"):
z = np.where(S > 0, (S - 1) / np.sqrt(var_s), np.where(S < 0, (S + 1) / np.sqrt(var_s), 0.0))
p = 2 * (1 - norm.cdf(np.abs(z)))
p = np.where(n >= 3, p, np.nan)
return S, var_s, z, p
def bh_fdr(p) -> np.ndarray:
"""Benjamini-Hochberg q-values; NaN p-values stay NaN and are not counted."""
p = np.asarray(p, dtype=float)
q = np.full_like(p, np.nan)
ok = np.isfinite(p)
pv = p[ok]
m = len(pv)
if m == 0:
return q
order = np.argsort(pv)
ranked = pv[order] * m / np.arange(1, m + 1)
ranked = np.minimum.accumulate(ranked[::-1])[::-1]
qv = np.empty(m)
qv[order] = np.minimum(ranked, 1.0)
q[ok] = qv
return q
def classify(slope, p_or_q, n, alpha: float = 0.05, min_years: int = 18) -> np.ndarray:
"""growth / decline / unreliable / insufficient, element-wise."""
slope = np.atleast_1d(np.asarray(slope, dtype=float))
pq = np.atleast_1d(np.asarray(p_or_q, dtype=float))
n = np.atleast_1d(np.asarray(n, dtype=float))
out = np.full(slope.shape, "unreliable", dtype=object)
sig = np.isfinite(pq) & (pq < alpha)
out[sig & (slope > 0)] = "growth"
out[sig & (slope < 0)] = "decline"
out[(n < min_years) | ~np.isfinite(slope)] = "insufficient"
return out
def per_decade(slope) -> np.ndarray:
return np.asarray(slope, dtype=float) * 10.0
def pct_per_decade(slope, mean) -> np.ndarray:
with np.errstate(invalid="ignore", divide="ignore"):
return np.asarray(slope, dtype=float) * 10.0 / np.asarray(mean, dtype=float) * 100.0
|