lsnu commited on
Commit
ec49bcd
·
verified ·
1 Parent(s): 34ffd87

Delete quick_eval_tri.py

Browse files
Files changed (1) hide show
  1. quick_eval_tri.py +0 -209
quick_eval_tri.py DELETED
@@ -1,209 +0,0 @@
1
- from __future__ import annotations
2
-
3
- import argparse
4
- import json
5
- import random
6
- import sys
7
- from pathlib import Path
8
-
9
- import numpy as np
10
- import torch
11
-
12
-
13
- def add_repo_to_path(repo_root: Path) -> None:
14
- repo_root = repo_root.resolve()
15
- if str(repo_root) not in sys.path:
16
- sys.path.insert(0, str(repo_root))
17
-
18
-
19
- def set_seed(seed: int) -> None:
20
- random.seed(seed)
21
- np.random.seed(seed)
22
- torch.manual_seed(seed)
23
- torch.set_num_threads(1)
24
- try:
25
- torch.set_num_interop_threads(1)
26
- except RuntimeError:
27
- pass
28
-
29
-
30
- def compute_mode_oracle_metrics(model, out, teacher_mode):
31
- teacher_mode_util, _, _, teacher_mode_selected = model.scorer(out.belief, out.planner, teacher_mode)
32
- pred_mode_branch = out.selected_branch_ids[:, 0]
33
- oracle_mode_branch = teacher_mode_selected[:, 0]
34
- counts = torch.bincount(oracle_mode_branch, minlength=model.cfg.num_branches)
35
- acc = (pred_mode_branch == oracle_mode_branch).float().mean().item()
36
- transfer_mask = oracle_mode_branch == 1
37
- if transfer_mask.any():
38
- transfer_recall = ((pred_mode_branch == 1) & transfer_mask).float().sum().item() / transfer_mask.float().sum().item()
39
- else:
40
- transfer_recall = float("nan")
41
- return {
42
- "mode_oracle_branch_counts": counts.tolist(),
43
- "mode_branch_acc_vs_oracle": acc,
44
- "mode_transfer_recall_vs_oracle": transfer_recall,
45
- "mode_oracle_utility_mean": teacher_mode_util.mean().item(),
46
- }
47
-
48
-
49
- def compute_candidate_oracle_metrics(model, out, teacher_candidates, target):
50
- teacher_util, _, teacher_branch_util, _ = model.scorer(out.belief, out.planner, teacher_candidates)
51
- oracle_best_full, _ = teacher_util.max(dim=1)
52
-
53
- hold_release_candidate_util = teacher_branch_util[..., [0, 2]].max(dim=-1).values
54
- oracle_best_hold_release, _ = hold_release_candidate_util.max(dim=1)
55
- transfer_branch_gain = oracle_best_full - oracle_best_hold_release
56
-
57
- if out.candidates.shape[1] > 3:
58
- keep = [i for i in range(out.candidates.shape[1]) if i != 3]
59
- oracle_best_without_transfer_template, _ = teacher_util[:, keep].max(dim=1)
60
- transfer_template_gain = oracle_best_full - oracle_best_without_transfer_template
61
- else:
62
- transfer_template_gain = torch.zeros_like(oracle_best_full)
63
-
64
- model_choice_teacher_util = teacher_util.gather(1, out.best_indices[:, None]).squeeze(1)
65
- ratio = model_choice_teacher_util.mean() / oracle_best_full.mean().clamp_min(1e-6)
66
-
67
- transfer_state_mask = target.expert_branch == 1
68
- mechanics_ids = target.mechanics
69
- scenario_transfer_fraction = {}
70
- for scenario_id, name in enumerate(model.cfg.mechanics_names):
71
- mask = mechanics_ids == scenario_id
72
- if mask.any():
73
- scenario_transfer_fraction[f"transfer_fraction_{name}"] = (
74
- (target.expert_branch[mask] == 1).float().mean().item()
75
- )
76
- else:
77
- scenario_transfer_fraction[f"transfer_fraction_{name}"] = float("nan")
78
-
79
- oracle_branch_counts = torch.bincount(target.expert_branch, minlength=model.cfg.num_branches)
80
-
81
- out_dict = {
82
- "oracle_branch_counts_from_labels": oracle_branch_counts.tolist(),
83
- "oracle_transfer_state_fraction": (target.expert_branch == 1).float().mean().item(),
84
- "oracle_best_full_mean": oracle_best_full.mean().item(),
85
- "oracle_best_hold_release_mean": oracle_best_hold_release.mean().item(),
86
- "oracle_transfer_branch_gain_mean": transfer_branch_gain.mean().item(),
87
- "oracle_transfer_branch_gain_transfer_states": transfer_branch_gain[transfer_state_mask].mean().item()
88
- if transfer_state_mask.any()
89
- else float("nan"),
90
- "oracle_transfer_template_gain_mean": transfer_template_gain.mean().item(),
91
- "oracle_transfer_template_gain_transfer_states": transfer_template_gain[transfer_state_mask].mean().item()
92
- if transfer_state_mask.any()
93
- else float("nan"),
94
- "teacher_utility_of_model_choice_mean": model_choice_teacher_util.mean().item(),
95
- "oracle_approx_ratio_of_model_choice": ratio.item(),
96
- "oracle_best_full_std": oracle_best_full.std(unbiased=False).item(),
97
- "teacher_model_choice_std": model_choice_teacher_util.std(unbiased=False).item(),
98
- }
99
- out_dict.update(scenario_transfer_fraction)
100
- return out_dict
101
-
102
-
103
- def train_and_eval(args: argparse.Namespace) -> dict:
104
- repo_root = Path(args.repo).resolve()
105
- add_repo_to_path(repo_root)
106
-
107
- from persist_vla import PersistVLA, PersistVLAConfig, SyntheticPersistDataset, TinyTokenizer, evaluate_chunks_oracle_rollout
108
- from persist_vla.training import compute_loss, compute_sequence_loss
109
-
110
- device = torch.device(args.device)
111
- set_seed(args.seed)
112
-
113
- cfg = PersistVLAConfig(
114
- hidden_dim=args.hidden_dim,
115
- num_candidates=args.num_candidates,
116
- sequence_horizon=max(1, args.sequence_horizon),
117
- chunk_horizon=args.chunk_horizon,
118
- )
119
- tok = TinyTokenizer()
120
- cfg.text_vocab_size = max(cfg.text_vocab_size, tok.vocab_size + 8)
121
- dataset = SyntheticPersistDataset(cfg=cfg, tokenizer=tok)
122
- model = PersistVLA(cfg).to(device)
123
- opt = torch.optim.Adam(model.parameters(), lr=args.lr)
124
- teacher = lambda s, c: evaluate_chunks_oracle_rollout(s, c, cfg)
125
-
126
- history = []
127
- for step in range(args.steps):
128
- model.train()
129
- if args.sequence_horizon > 1:
130
- traj = dataset.sample_trajectory_batch(args.batch_size, horizon=args.sequence_horizon, device=device)
131
- losses = compute_sequence_loss(model, traj, oracle_evaluator=teacher)
132
- else:
133
- obs, sup = dataset.sample_batch(args.batch_size, device=device)
134
- _, losses = compute_loss(model, obs, sup, oracle_evaluator=teacher)
135
- opt.zero_grad(set_to_none=True)
136
- losses["total"].backward()
137
- if args.grad_clip > 0:
138
- torch.nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip)
139
- opt.step()
140
- if (step % max(1, args.log_every) == 0) or (step == args.steps - 1):
141
- snapshot = {k: float(v.detach().cpu()) for k, v in losses.items() if k in {
142
- "total", "action", "branch_prior", "branch_choice", "world_hold", "world_transfer", "world_release", "mechanics"
143
- }}
144
- snapshot["step"] = step
145
- history.append(snapshot)
146
-
147
- model.eval()
148
- with torch.no_grad():
149
- eval_obs, eval_target = dataset.sample_batch(args.eval_batch_size, device=device)
150
- out, losses = compute_loss(model, eval_obs, eval_target, oracle_evaluator=teacher)
151
- teacher_mode = model.teacher_metrics_from_oracle(teacher(eval_obs.privileged_state, out.decoded.mode_chunk.unsqueeze(1)))
152
- teacher_candidates = model.teacher_metrics_from_oracle(teacher(eval_obs.privileged_state, out.candidates))
153
- mode_metrics = compute_mode_oracle_metrics(model, out, teacher_mode)
154
- cand_metrics = compute_candidate_oracle_metrics(model, out, teacher_candidates, eval_target)
155
-
156
- result = {
157
- "train_trace": history,
158
- "eval_total_loss": float(losses["total"].detach().cpu()),
159
- "eval_action_loss": float(losses["action"].detach().cpu()),
160
- "eval_branch_prior_loss": float(losses["branch_prior"].detach().cpu()),
161
- "eval_branch_choice_loss": float(losses["branch_choice"].detach().cpu()),
162
- "eval_world_transfer_loss": float(losses["world_transfer"].detach().cpu()),
163
- "eval_world_release_loss": float(losses["world_release"].detach().cpu()),
164
- "config": {
165
- "steps": args.steps,
166
- "batch_size": args.batch_size,
167
- "eval_batch_size": args.eval_batch_size,
168
- "hidden_dim": args.hidden_dim,
169
- "lr": args.lr,
170
- "seed": args.seed,
171
- "sequence_horizon": args.sequence_horizon,
172
- "num_candidates": args.num_candidates,
173
- "chunk_horizon": args.chunk_horizon,
174
- },
175
- }
176
- result.update(mode_metrics)
177
- result.update(cand_metrics)
178
- return result
179
-
180
-
181
- def main() -> None:
182
- parser = argparse.ArgumentParser(description="Fast train-and-eval harness for PERSIST-VLA-Tri")
183
- parser.add_argument("--repo", type=str, required=True, help="Path to persist_vla_reference repo root")
184
- parser.add_argument("--device", type=str, default="cpu")
185
- parser.add_argument("--steps", type=int, default=80)
186
- parser.add_argument("--batch-size", type=int, default=24)
187
- parser.add_argument("--eval-batch-size", type=int, default=256)
188
- parser.add_argument("--hidden-dim", type=int, default=48)
189
- parser.add_argument("--lr", type=float, default=3e-3)
190
- parser.add_argument("--seed", type=int, default=0)
191
- parser.add_argument("--sequence-horizon", type=int, default=1)
192
- parser.add_argument("--num-candidates", type=int, default=6)
193
- parser.add_argument("--chunk-horizon", type=int, default=4)
194
- parser.add_argument("--grad-clip", type=float, default=1.0)
195
- parser.add_argument("--log-every", type=int, default=20)
196
- parser.add_argument("--output-json", type=str, default="")
197
- args = parser.parse_args()
198
-
199
- result = train_and_eval(args)
200
- text = json.dumps(result, indent=2)
201
- print(text)
202
- if args.output_json:
203
- out_path = Path(args.output_json)
204
- out_path.parent.mkdir(parents=True, exist_ok=True)
205
- out_path.write_text(text)
206
-
207
-
208
- if __name__ == "__main__":
209
- main()