File size: 7,627 Bytes
c8d5563
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
"""pi05_expert generator vs OpenPI's own Pi0 (pi05=True), on CPU with the dummy Gemma variants.

1. Every Pi0 param exists in the AdaSpeVLA expert model with the same name and shape (what PluginWeightLoader
   takes from pi05_base).
2. With Pi0's weights copied in (and cp_knot_proj at its zero init), the expert's control-point velocity equals Pi0's
   action velocity from Pi0.sample_actions on the same observation and noise (1 ODE step), and 10-step sampling
   matches too. So tokens, adaRMS time conditioning, attention masks, positions and KV-cache use are pi0.5's.
3. cp_knot_proj is wired in (a nonzero kernel changes the velocity).
4. With knot_loss_coeff=0 the flow loss alone still reaches the backbone (gradients through the KV cache).
Usage: TASK=insert_tube python adaspe_train/tests/test_expert_parity.py
"""
import os, sys; os.environ.setdefault("JAX_PLATFORMS", "cpu")  # noqa: E401,E702
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))  # adaspe_train/
import dataclasses  # noqa: E402

import flax.nnx as nnx  # noqa: E402
import flax.traverse_util as tu  # noqa: E402
import jax  # noqa: E402
import jax.numpy as jnp  # noqa: E402
import numpy as np  # noqa: E402
from openpi.models import model as _model  # noqa: E402
from openpi.models import pi0_config  # noqa: E402

import tasks  # noqa: E402
from adaspe.model import AdaSpePi05Config  # noqa: E402

H, K, A = tasks.ACTION_HORIZON, tasks.NUM_CONTROL_POINTS, 7
cfg = AdaSpePi05Config(
    paligemma_variant="dummy", action_expert_variant="dummy", generator="pi05_expert", action_horizon=H,
    num_control_points=K, min_query_interval=tasks.MIN_QUERY_INTERVAL, max_query_interval=tasks.MAX_QUERY_INTERVAL,
    timing_label_mode=tasks.LABEL_MODE, knot_label_mode=tasks.LABEL_MODE, knot_motion_norm=tasks.KNOT_MOTION_NORM,
    timing_motion_scale=(0.02,) * 6, e_crit=10.0, num_ode_steps=10, timing_num_ode_steps=10)
ours = cfg.create(jax.random.key(0))
ref = pi0_config.Pi0Config(pi05=True, paligemma_variant="dummy", action_expert_variant="dummy", action_horizon=K
                           ).create(jax.random.key(1))

# 1. names / shapes, then copy Pi0's weights into ours. Layer lists have int keys: compare on "/"-joined names
# (as PluginWeightLoader does) but write back with the model's own tuple keys.
def flat(s):
    return {"/".join(map(str, k)): v for k, v in tu.flatten_dict(s.to_pure_dict()).items()}


def set_params(model, values):
    """Overwrite the model params whose "/"-joined name is in `values`."""
    st = nnx.state(model, nnx.Param)
    d = tu.flatten_dict(st.to_pure_dict())
    for k in d:
        if "/".join(map(str, k)) in values:
            d[k] = values["/".join(map(str, k))]
    st.replace_by_pure_dict(tu.unflatten_dict(d))
    nnx.update(model, st)


rf, of = flat(nnx.state(ref, nnx.Param)), flat(nnx.state(ours, nnx.Param))
missing = [k for k in rf if k not in of]
bad = [k for k in rf if k in of and np.shape(rf[k]) != np.shape(of[k])]
assert not missing and not bad, (missing[:5], bad[:5])
extra = sorted({k.split("/")[0] for k in of if k not in rf})
print(f"1. all {len(rf)} Pi0 arrays present with equal shapes; AdaSpeVLA-only modules: {extra}")
set_params(ours, rf)

obs = cfg.fake_obs(2)
noise = jax.random.normal(jax.random.key(3), (2, K, A))
noise32 = jnp.pad(noise, ((0, 0), (0, 0), (0, cfg.action_dim - A)))  # pi0.5 action padding


def ours_sample(model, steps):
    o = _model.preprocess_observation(None, obs, train=False)
    cond, ctx = model.encode(o)
    hz = model._horizon(2)
    knots, _ = model.predict_knots(cond, hz)
    return model.sample_control_points(None, cond, knots, hz, ctx, steps, noise=noise)


# 2a. inference path at t=1 vs the real Pi0.sample_actions (1 ODE step: x0 = noise - v  =>  v = noise - x0)
v_ref = (noise32 - ref.sample_actions(jax.random.key(4), obs, num_steps=1, noise=noise32))[..., :A]
v_our = noise - ours_sample(ours, 1)
err1 = float(jnp.max(jnp.abs(v_our - v_ref)) / jnp.max(jnp.abs(v_ref)))
print(f"2a. velocity at t=1 vs Pi0.sample_actions: max rel. diff {err1:.2e}")
assert err1 < 1e-3, err1

# 2b. training path at t=0.37: ours (prefix pass -> KV cache -> expert pass) vs Pi0.compute_loss's single joint pass
from openpi.models.pi0 import make_attn_mask  # noqa: E402

o = _model.preprocess_observation(None, obs, train=False)
t = jnp.full((2,), 0.37)
pt, pm, par = ref.embed_prefix(o)
st_, sm, sar, ada = ref.embed_suffix(o, noise32, t)
im, am = jnp.concatenate([pm, sm], 1), jnp.concatenate([par, sar], 0)
(_, so), _ = ref.PaliGemma.llm([pt, st_], mask=make_attn_mask(im, am), positions=jnp.cumsum(im, 1) - 1,
                               adarms_cond=[None, ada])
v_joint = ref.action_out_proj(so[:, -K:])[..., :A]
cond, ctx = ours.encode(o)
knots, _ = ours.predict_knots(cond, ours._horizon(2))
v_two = ours._expert_velocity(noise, t, ours._cp_token_bias(knots, ours._horizon(2)), ctx)
err2 = float(jnp.max(jnp.abs(v_two - v_joint)) / jnp.max(jnp.abs(v_joint)))
print(f"2b. velocity at t=0.37, two-pass (ours) vs Pi0 joint pass: max rel. diff {err2:.2e}")
assert err2 < 1e-2, err2

# 3. the knot-position projection is used
fl = flat(nnx.state(ours, nnx.Param))
fl["cp_knot_proj/kernel"] = np.asarray(jax.random.normal(jax.random.key(5), np.shape(fl["cp_knot_proj/kernel"])) * 0.1)
probe = cfg.create(jax.random.key(0))
set_params(probe, fl)
d3 = float(jnp.max(jnp.abs((noise - ours_sample(probe, 1)) - v_our)))
print(f"3. nonzero cp_knot_proj changes the velocity by {d3:.3e}")
assert d3 > 1e-4

# 4. flow loss alone reaches the backbone through the KV cache. A fresh pi0.5 has zero-init adaRMS modulation, so every
# expert residual is gated by 0 (gemma._gated_residual: x + y * gate) and attention to the prefix is cut off; pi05_base's
# trained gates are nonzero (checked in 5). So give the modulation small random weights first.
cfg0 = dataclasses.replace(cfg, knot_loss_coeff=0.0)
m0 = cfg0.create(jax.random.key(0))
f0 = flat(nnx.state(m0, nnx.Param))
mod = [k for k in f0 if k.startswith("PaliGemma/llm") and "norm_1/Dense_0/kernel" in k]
assert mod, "no adaRMS modulation kernels found"
set_params(m0, {k: np.asarray(jax.random.normal(jax.random.key(6 + i), np.shape(f0[k]))) * 0.02 for i, k in enumerate(mod)})
act = cfg0.fake_act(2)
g, p, r = nnx.split(m0, nnx.Param, ...)
grads = jax.grad(lambda p_: nnx.merge(g, p_, r).compute_loss(jax.random.key(1), obs, act, train=True).mean())(p)
gf = {k: float(jnp.abs(v).sum()) for k, v in flat(grads).items()}
bb = sum(v for k, v in gf.items() if k.startswith("PaliGemma/") and "_1" not in k)
exp = sum(v for k, v in gf.items() if k.startswith("PaliGemma/llm") and "_1" in k)
print(f"4. knot loss off: |grad| backbone {bb:.3e}, expert {exp:.3e}, cp_knot_proj {gf['cp_knot_proj/kernel']:.3e}")
assert bb > 0 and exp > 0 and gf["cp_knot_proj/kernel"] > 0

# 5. pi05_base's adaRMS modulation (what the real run loads) is nonzero, so the KV-cache path is open from step 0.
base = f"{tasks.ROOT}/.cache/openpi/openpi-assets/checkpoints/pi05_base/params"
if os.path.isdir(base):
    loaded = {"/".join(map(str, k)): v for k, v in tu.flatten_dict(
        _model.restore_params(base, restore_type=np.ndarray)).items()}
    gates = {k: v for k, v in loaded.items() if k.startswith("PaliGemma/llm") and "norm_1/Dense_0/kernel" in k}
    stats = {k.split("/")[-3]: f"mean|w| {np.abs(v).mean():.2e}, zero {np.mean(v == 0) * 100:.1f}%" for k, v in gates.items()}
    print(f"5. pi05_base adaRMS modulation kernels: {stats}")
    assert gates and all(np.abs(v).mean() > 1e-6 for v in gates.values())
else:
    print(f"5. skipped: {base} not found")
print("expert parity OK")