File size: 6,156 Bytes
7e6579d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
"""Trend statistics, shared by the formula (F3) and the ML session's residual trends.

One implementation, copied (not forked) into the ML repo.

- trend_1d:    Sen slope + Mann-Kendall (Hamed-Rao autocorrelation correction by default) + 95% CI (Gilbert 1987).
- trend_many:  trend_1d over rows of a 2-D array (joblib).
- mk_vectorized / sen_vectorized: fast numpy versions (original MK, no autocorrelation correction),
               used for placebo permutations and sensitivity loops.
- bh_fdr:      Benjamini-Hochberg q-values.
- classify:    growth / decline / unreliable / insufficient.
"""
from __future__ import annotations

import math
import warnings

import numpy as np
import pymannkendall as pmk
from scipy.stats import norm

CLASSES = ("growth", "decline", "unreliable", "insufficient")


def _sen_ci(y: np.ndarray, t: np.ndarray, alpha: float = 0.05) -> tuple[float, float, float]:
    """Sen slope and its (1-alpha) CI by the rank method (Gilbert 1987). y, t without NaN."""
    n = len(y)
    i, j = np.triu_indices(n, k=1)
    dt = t[j] - t[i]
    slopes = np.sort((y[j] - y[i]) / dt)
    m = len(slopes)
    med = float(np.median(slopes))
    # variance of S without tie correction (ties are rare for float NDVI means)
    var_s = n * (n - 1) * (2 * n + 5) / 18.0
    c = norm.ppf(1 - alpha / 2) * math.sqrt(var_s)
    lo_idx = int(math.floor((m - c) / 2.0))
    hi_idx = int(math.ceil((m + c) / 2.0))
    lo_idx = min(max(lo_idx, 0), m - 1)
    hi_idx = min(max(hi_idx, 0), m - 1)
    return med, float(slopes[lo_idx]), float(slopes[hi_idx])


def trend_1d(y, years, method: str = "hamed_rao", alpha: float = 0.05, min_n: int = 8) -> dict:
    """Trend of one series. NaNs are dropped. Returns slope per time unit (per year)."""
    y = np.asarray(y, dtype=float)
    t = np.asarray(years, dtype=float)
    ok = np.isfinite(y)
    y, t = y[ok], t[ok]
    n = int(len(y))
    out = {"slope": np.nan, "intercept": np.nan, "ci_lo": np.nan, "ci_hi": np.nan,
           "p": np.nan, "z": np.nan, "tau": np.nan, "n": n, "method": method}
    if n < min_n or np.nanstd(y) == 0:
        if n >= min_n:
            out.update(slope=0.0, intercept=float(np.median(y)), ci_lo=0.0, ci_hi=0.0, p=1.0, z=0.0, tau=0.0)
        return out
    slope, lo, hi = _sen_ci(y, t, alpha)
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        if method == "hamed_rao":
            r = pmk.hamed_rao_modification_test(y)
        else:
            r = pmk.original_test(y)
    p = float(r.p)
    if not np.isfinite(p):  # Hamed-Rao can fail on degenerate series; fall back to original MK
        r = pmk.original_test(y)
        p = float(r.p)
    intercept = float(np.median(y - slope * t))
    out.update(slope=slope, intercept=intercept, ci_lo=lo, ci_hi=hi, p=p, z=float(r.z), tau=float(r.Tau))
    return out


def _trend_rows(Y, years, method, alpha, min_n):
    return [trend_1d(row, years, method, alpha, min_n) for row in Y]


def trend_many(Y, years, method: str = "hamed_rao", alpha: float = 0.05, min_n: int = 8,
               n_jobs: int = -1, chunk: int = 2000):
    """trend_1d for every row of Y (n_series, n_years). Returns a dict of 1-D arrays."""
    import pandas as pd
    from joblib import Parallel, delayed

    Y = np.asarray(Y, dtype=float)
    chunks = [Y[i:i + chunk] for i in range(0, len(Y), chunk)]
    if len(chunks) <= 1 or n_jobs == 1:
        rows = _trend_rows(Y, years, method, alpha, min_n)
    else:
        parts = Parallel(n_jobs=n_jobs)(delayed(_trend_rows)(c, years, method, alpha, min_n) for c in chunks)
        rows = [r for p in parts for r in p]
    df = pd.DataFrame(rows)
    return {k: df[k].to_numpy() for k in df.columns}


def sen_vectorized(Y, years) -> np.ndarray:
    """Sen slope for each row (NaN-aware)."""
    Y = np.asarray(Y, dtype=np.float32)
    t = np.asarray(years, dtype=np.float32)
    i, j = np.triu_indices(len(t), k=1)
    s = (Y[:, j] - Y[:, i]) / (t[j] - t[i])
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        return np.nanmedian(s, axis=1)


def mk_vectorized(Y, years=None):
    """Original Mann-Kendall for each row (NaN-aware, no tie/autocorrelation correction).

    Returns (S, var_s, z, p)."""
    Y = np.asarray(Y, dtype=np.float32)
    n_t = Y.shape[1]
    i, j = np.triu_indices(n_t, k=1)
    d = Y[:, j] - Y[:, i]
    valid = np.isfinite(d)
    S = np.where(valid, np.sign(d), 0).sum(axis=1).astype(float)
    n = np.isfinite(Y).sum(axis=1).astype(float)
    var_s = n * (n - 1) * (2 * n + 5) / 18.0
    with np.errstate(invalid="ignore", divide="ignore"):
        z = np.where(S > 0, (S - 1) / np.sqrt(var_s), np.where(S < 0, (S + 1) / np.sqrt(var_s), 0.0))
    p = 2 * (1 - norm.cdf(np.abs(z)))
    p = np.where(n >= 3, p, np.nan)
    return S, var_s, z, p


def bh_fdr(p) -> np.ndarray:
    """Benjamini-Hochberg q-values; NaN p-values stay NaN and are not counted."""
    p = np.asarray(p, dtype=float)
    q = np.full_like(p, np.nan)
    ok = np.isfinite(p)
    pv = p[ok]
    m = len(pv)
    if m == 0:
        return q
    order = np.argsort(pv)
    ranked = pv[order] * m / np.arange(1, m + 1)
    ranked = np.minimum.accumulate(ranked[::-1])[::-1]
    qv = np.empty(m)
    qv[order] = np.minimum(ranked, 1.0)
    q[ok] = qv
    return q


def classify(slope, p_or_q, n, alpha: float = 0.05, min_years: int = 18) -> np.ndarray:
    """growth / decline / unreliable / insufficient, element-wise."""
    slope = np.atleast_1d(np.asarray(slope, dtype=float))
    pq = np.atleast_1d(np.asarray(p_or_q, dtype=float))
    n = np.atleast_1d(np.asarray(n, dtype=float))
    out = np.full(slope.shape, "unreliable", dtype=object)
    sig = np.isfinite(pq) & (pq < alpha)
    out[sig & (slope > 0)] = "growth"
    out[sig & (slope < 0)] = "decline"
    out[(n < min_years) | ~np.isfinite(slope)] = "insufficient"
    return out


def per_decade(slope) -> np.ndarray:
    return np.asarray(slope, dtype=float) * 10.0


def pct_per_decade(slope, mean) -> np.ndarray:
    with np.errstate(invalid="ignore", divide="ignore"):
        return np.asarray(slope, dtype=float) * 10.0 / np.asarray(mean, dtype=float) * 100.0