File size: 4,728 Bytes
2188a91 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 | """Claim 5 audit: recompute the reported average rank of 1.24 from Table 1 itself.
Table 1 is parsed straight out of the arXiv HTML text dump (paper/paper.txt) so the
numbers are not re-typed by hand. We then rank the 12 methods on every
(dataset, metric) cell and report the mean rank of "Ours" under several readings of
the claim, since the paper is ambiguous about which metrics enter the average
(Sec 4.1 says "across all metrics", the challenge statement says accuracy/F1/AUROC).
"""
import json
import re
import numpy as np
METHODS = ["Autoformer", "Crossformer", "FEDformer", "Informer", "iTransformer",
"MTST", "Nonformer", "PatchTST", "Reformer", "Transformer",
"Medformer", "Ours"]
DATASETS = ["ADFTD", "PTB", "PTB-XL", "APAVA", "SleepEDF", "FLAAP", "UCI-HAR"]
METRICS = ["Accuracy", "Precision", "Recall", "F1-score", "AUROC"]
def parse(path="paper/paper.txt"):
txt = open(path, encoding="utf-8").read()
# Table 1 spans from the header block to the "Table 1 :" caption
start = txt.index(" Dataset \n Metrics \n Autoformer")
end = txt.index(" Table 1 : ")
body = txt[start:end]
# every "value ± std" pair, in reading order
vals = [float(m) for m in re.findall(r"(\d+\.\d+)\s*±", body)]
assert len(vals) == 7 * 5 * 12, f"expected 420 numbers, parsed {len(vals)}"
arr = np.array(vals).reshape(7, 5, 12)
return arr
def ranks_for_ours(arr, metric_idx):
"""Rank (1 = best, higher value is better for every metric here) of 'Ours'."""
out = {}
for di, ds in enumerate(DATASETS):
for mi in metric_idx:
col = arr[di, mi]
# competition ranking with ties sharing the average rank
order = (-col).argsort()
r = np.empty(12)
r[order] = np.arange(1, 13)
# handle exact ties
for v in np.unique(col):
m = col == v
if m.sum() > 1:
r[m] = r[m].mean()
out[(ds, METRICS[mi])] = float(r[METHODS.index("Ours")])
return out
def main():
arr = parse()
report = {"table_shape": list(arr.shape)}
# spot-check a few known cells against the paper text
checks = {
("ADFTD", "Accuracy", "Ours"): 53.91,
("ADFTD", "F1-score", "Ours"): 51.79,
("ADFTD", "F1-score", "Medformer"): 50.65,
("PTB-XL", "F1-score", "Ours"): 63.51,
("UCI-HAR", "Accuracy", "Medformer"): 91.65,
("SleepEDF", "AUROC", "Ours"): 95.37,
}
ok = {}
for (ds, me, mo), want in checks.items():
got = arr[DATASETS.index(ds), METRICS.index(me), METHODS.index(mo)]
ok[f"{ds}/{me}/{mo}"] = {"parsed": float(got), "paper": want,
"match": bool(abs(got - want) < 1e-9)}
report["parse_spot_checks"] = ok
variants = {
"all_5_metrics": [0, 1, 2, 3, 4],
"acc_f1_auroc (challenge wording)": [0, 3, 4],
"acc_f1": [0, 3],
}
for name, idx in variants.items():
r = ranks_for_ours(arr, idx)
vals = np.array(list(r.values()))
report[name] = {
"n_cells": int(vals.size),
"avg_rank_ours": float(vals.mean()),
"n_rank1": int((vals == 1).sum()),
"worst_rank": float(vals.max()),
"cells_not_rank1": {f"{k[0]}/{k[1]}": v for k, v in r.items() if v != 1},
}
# average rank of every method under the 5-metric reading, for context
allranks = {}
for mo in METHODS:
rs = []
for di in range(7):
for mi in range(5):
col = arr[di, mi]
order = (-col).argsort()
rr = np.empty(12)
rr[order] = np.arange(1, 13)
for v in np.unique(col):
m = col == v
if m.sum() > 1:
rr[m] = rr[m].mean()
rs.append(rr[METHODS.index(mo)])
allranks[mo] = float(np.mean(rs))
report["avg_rank_all_methods_5metrics"] = allranks
# one-tailed Wilcoxon signed-rank, Ours vs Medformer (paper's stated test)
from scipy.stats import wilcoxon
ours = arr[:, :, METHODS.index("Ours")].ravel()
med = arr[:, :, METHODS.index("Medformer")].ravel()
st, p = wilcoxon(ours, med, alternative="greater")
report["wilcoxon_ours_gt_medformer"] = {"statistic": float(st), "p_value": float(p),
"n_pairs": int(ours.size),
"ours_wins": int((ours > med).sum())}
print(json.dumps(report, indent=2))
with open("results/table1_rank_audit.json", "w") as f:
json.dump(report, f, indent=2)
if __name__ == "__main__":
main()
|