File size: 3,521 Bytes
216004d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 | #!/usr/bin/env python3
"""IDBD vs 手工 value 的最终联合分析 (最差遗忘界 = 主判据)。"""
import glob
import json
import math
import os
import sys
import numpy as np
ROOT = sys.argv[1] if len(sys.argv) > 1 else "/work/liuyuanjie/headless"
def tsf(x, d):
if x <= 0:
return 0.5 + (0.5 - tsf(-x, d))
c = math.gamma((d + 1) / 2) / (math.sqrt(d * math.pi) * math.gamma(d / 2))
n, hi = 12000, x + 60.0
h = (hi - x) / n
s = 0.0
for i in range(n + 1):
tt = x + i * h
w = 1 if i in (0, n) else (4 if i % 2 else 2)
s += w * c * (1 + tt * tt / d) ** (-(d + 1) / 2)
return s * h / 3
def welch(m1, s1, n1, m2, s2, n2, lab):
se = math.sqrt(s1 ** 2 / n1 + s2 ** 2 / n2)
t = (m1 - m2) / se
df = se ** 4 / ((s1 ** 2 / n1) ** 2 / (n1 - 1) + (s2 ** 2 / n2) ** 2 / (n2 - 1))
p = 2 * tsf(abs(t), df)
print(" %-34s Δ=%+.4f t=%6.2f df=%4.1f p=%.4f %s"
% (lab, m1 - m2, t, df, p, "**显著**" if p < 0.05 else "不显著"))
return p
def get(pats):
at, wf = [], []
for pat in pats:
for f in glob.glob(os.path.join(ROOT, "results", pat, "lm4_wave_results.json")):
try:
d = json.load(open(f))
except Exception:
continue
v = d.get("replay")
if isinstance(v, dict) and v.get("any_time_acc") is not None:
at.append(v["any_time_acc"])
wf.append(v["worst_case_forget"])
return np.array(at), np.array(wf)
S8 = (11, 12, 13, 14, 15, 16)
RLX = (1, 2, 3, 4, 5, 6)
A, WA = get(["s8_idbd_s%d" % s for s in S8])
B, WB = get(["s8_value_s%d" % s for s in S8])
C, WC = get(["rlx_idbd_s%d" % s for s in RLX])
D, WD = get(["rlx_value_s%d" % s for s in RLX])
print("=" * 96)
print("IDBD vs 手工 value —— 主判据: 最差遗忘界 (越小越好)")
print("=" * 96)
print()
print("--- 新一批 s8 (同代码版本, 各 n=%d) ---" % len(A))
print(" idbd-raw any_time %.4f±%.4f 最差遗忘 %.4f±%.4f"
% (A.mean(), A.std(ddof=1) if len(A) > 1 else 0,
WA.mean(), WA.std(ddof=1) if len(WA) > 1 else 0))
print(" value any_time %.4f±%.4f 最差遗忘 %.4f±%.4f"
% (B.mean(), B.std(ddof=1) if len(B) > 1 else 0,
WB.mean(), WB.std(ddof=1) if len(WB) > 1 else 0))
welch(A.mean(), A.std(ddof=1), len(A), B.mean(), B.std(ddof=1), len(B),
"any_time: idbd-raw vs value")
p1 = welch(WA.mean(), WA.std(ddof=1), len(A), WB.mean(), WB.std(ddof=1), len(B),
"最差遗忘: idbd-raw vs value")
print()
A2 = np.concatenate([A, C])
WA2 = np.concatenate([WA, WC])
B2 = np.concatenate([B, D])
WB2 = np.concatenate([WB, WD])
print("--- 合并两批 s8 + rlx (各 n=%d) ---" % len(A2))
print(" IDBD 系 any_time %.4f±%.4f 最差遗忘 %.4f±%.4f"
% (A2.mean(), A2.std(ddof=1), WA2.mean(), WA2.std(ddof=1)))
print(" value 系 any_time %.4f±%.4f 最差遗忘 %.4f±%.4f"
% (B2.mean(), B2.std(ddof=1), WB2.mean(), WB2.std(ddof=1)))
p_a = welch(A2.mean(), A2.std(ddof=1), len(A2), B2.mean(), B2.std(ddof=1), len(B2),
"any_time: IDBD系 vs value系")
p2 = welch(WA2.mean(), WA2.std(ddof=1), len(A2), WB2.mean(), WB2.std(ddof=1), len(B2),
"最差遗忘: IDBD系 vs value系")
print()
print("=== 判读 ===")
print(" 最差遗忘界: 单批 p=%.3f, 合并 p=%.3f" % (p1, p2))
print(" IDBD 系 %.4f vs value 系 %.4f -> %+.1f%%"
% (WA2.mean(), WB2.mean(), 100 * (WA2.mean() / WB2.mean() - 1)))
print(" any_time : 合并 p=%.3f" % p_a)
|