Add files using upload-large-folder tool
Browse files- model/tachin_paired30_cache_20260923/README.md +85 -0
- model/tachin_paired30_cache_20260923/SHA256SUMS +264 -0
- model/tachin_paired30_cache_20260923/audit/cache_audit.json +57 -0
- model/tachin_paired30_cache_20260923/audit/comparison.json +155 -0
- model/tachin_paired30_cache_20260923/cache/complete.json +21 -0
- model/tachin_paired30_cache_20260923/code/scripts/audit_tachin_paired30.py +78 -0
- model/tachin_paired30_cache_20260923/code/scripts/build_tachin_paired30_cache.py +174 -0
- model/tachin_paired30_cache_20260923/code/scripts/cache_tachin_adapted_base.py +251 -0
- model/tachin_paired30_cache_20260923/code/scripts/cache_tachin_twla_w3.py +299 -0
- model/tachin_paired30_cache_20260923/code/scripts/summarize_tachin_paired30.py +77 -0
- model/tachin_paired30_cache_20260923/code/scripts/train_tachin_table1_editor.py +539 -0
- model/tachin_paired30_cache_20260923/code/src/tactile_editor/action_layout.py +123 -0
- model/tachin_paired30_cache_20260923/code/src/tactile_editor/action_objectives.py +233 -0
- model/tachin_paired30_cache_20260923/code/src/tactile_editor/action_output.py +71 -0
- model/tachin_paired30_cache_20260923/code/src/tactile_editor/architecture_ablation.py +423 -0
- model/tachin_paired30_cache_20260923/code/src/tactile_editor/auxiliary_objectives.py +264 -0
- model/tachin_paired30_cache_20260923/code/src/tactile_editor/config.py +160 -0
- model/tachin_paired30_cache_20260923/code/src/tactile_editor/contact_filter.py +91 -0
- model/tachin_paired30_cache_20260923/code/src/tactile_editor/direct_residual.py +327 -0
- model/tachin_paired30_cache_20260923/code/src/tactile_editor/editor_backends.py +196 -0
- model/tachin_paired30_cache_20260923/code/src/tactile_editor/editor_interface.py +96 -0
- model/tachin_paired30_cache_20260923/code/src/tactile_editor/encoder.py +209 -0
- model/tachin_paired30_cache_20260923/code/src/tactile_editor/full_cache.py +86 -0
- model/tachin_paired30_cache_20260923/code/src/tactile_editor/fusion.py +211 -0
- model/tachin_paired30_cache_20260923/code/src/tactile_editor/informative_data.py +177 -0
- model/tachin_paired30_cache_20260923/code/src/tactile_editor/memoryvla_baseline.py +104 -0
- model/tachin_paired30_cache_20260923/code/src/tactile_editor/model_adapters.py +287 -0
- model/tachin_paired30_cache_20260923/code/src/tactile_editor/native_tachin.py +201 -0
- model/tachin_paired30_cache_20260923/code/src/tactile_editor/rgb_world_editor.py +419 -0
- model/tachin_paired30_cache_20260923/code/src/tactile_editor/runtime.py +376 -0
- model/tachin_paired30_cache_20260923/code/src/tactile_editor/touch_study.py +196 -0
- model/tachin_paired30_cache_20260923/code/src/tactile_editor/trainability.py +99 -0
- model/tachin_paired30_cache_20260923/code/src/tactile_editor/training.py +180 -0
- model/tachin_paired30_cache_20260923/code/src/tactile_editor/unified_cache.py +122 -0
- model/tachin_paired30_cache_20260923/code/src/tactile_editor/visual_dynamics_pilot.py +215 -0
- model/tachin_paired30_cache_20260923/code/src/tactile_editor/world_residual_probe.py +362 -0
- model/tachin_paired30_cache_20260923/code/tests/test_tachin_native_editor.py +63 -0
- model/tachin_paired30_cache_20260923/environment.txt +7 -0
- model/twla_tachin_common_adaptation_20260922/README.md +45 -0
- model/twla_tachin_common_adaptation_20260922/SHA256SUMS +13 -0
- model/twla_tachin_common_adaptation_20260922/code/adapt_twla_tachin_common.py +374 -0
- model/twla_tachin_common_adaptation_20260922/code/cache_tachin_twla_w3.py +299 -0
- model/twla_tachin_common_adaptation_20260922/code/rebuild_twla_adapted_checkpoint.py +96 -0
- model/twla_tachin_common_adaptation_20260922/code/verify_twla_tachin_adaptation.py +128 -0
- model/twla_tachin_common_adaptation_20260922/provenance/COMPLETE.json +12 -0
- model/twla_tachin_common_adaptation_20260922/provenance/config.json +33 -0
- model/twla_tachin_common_adaptation_20260922/provenance/initial_config_used.json +115 -0
- model/twla_tachin_common_adaptation_20260922/provenance/initial_ten_domain_config.json +115 -0
- model/twla_tachin_common_adaptation_20260922/provenance/metrics.jsonl +0 -0
- model/twla_tachin_common_adaptation_20260922/provenance/verification.json +15 -0
model/tachin_paired30_cache_20260923/README.md
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Tachin paired-30 policy caches and tactile-editor artifacts
|
| 2 |
+
|
| 3 |
+
This release contains the cached outputs and editor artifacts used for the Tachin
|
| 4 |
+
Table 1 experiments on 2026-09-23. The policy checkpoints were first adapted on
|
| 5 |
+
the Tachin training split. SmolVLA, VITRA, and T-Rex use the adaptation
|
| 6 |
+
checkpoints already published in `model/tachin_adaptation_20260922`. T-WLA uses
|
| 7 |
+
the separately published `model/twla_tachin_common_adaptation_20260922`.
|
| 8 |
+
|
| 9 |
+
## Protocol
|
| 10 |
+
|
| 11 |
+
- Dataset split: the same fixed Tachin train/validation/test episode split for
|
| 12 |
+
every method.
|
| 13 |
+
- Plan/vision rate: 10 Hz.
|
| 14 |
+
- Tactile/editor rate: 30 Hz, with causal decision times at 0, 33, and 67 ms
|
| 15 |
+
after each plan time.
|
| 16 |
+
- Action rate: 80 Hz; each prediction contains eight future actions (100 ms).
|
| 17 |
+
- The first editable action indices for the three tactile decisions are 0, 3,
|
| 18 |
+
and 6.
|
| 19 |
+
- Tactile input is not threshold-gated in this release (`gated=false`).
|
| 20 |
+
- Normalization statistics use only the original Tachin training split.
|
| 21 |
+
- The held-out test split was used only for final evaluation, not checkpoint
|
| 22 |
+
selection.
|
| 23 |
+
|
| 24 |
+
The cache audit reports 18,208/1,006/1,169 plans and
|
| 25 |
+
49,744/2,655/2,433 paired 30-Hz rows for train/validation/test. See
|
| 26 |
+
`audit/cache_audit.json`; its final status is `PASS`.
|
| 27 |
+
|
| 28 |
+
## Contents
|
| 29 |
+
|
| 30 |
+
- `cache/shared/{train,val,test}`: aligned tactile pressure, masks, action
|
| 31 |
+
targets, MANO beta, row identity, and causal future-pressure arrays shared by
|
| 32 |
+
all methods.
|
| 33 |
+
- `cache/bases/{smolvla,vitra,trex}`: cached action predictions from the three
|
| 34 |
+
Tachin-adapted baseline policies.
|
| 35 |
+
- `cache/bases/twla`: T-WLA W3 cached action predictions plus image tokens,
|
| 36 |
+
visual memory, clean RGB world-model latents, ROI targets, and validity
|
| 37 |
+
metadata. Each split has two shards.
|
| 38 |
+
- `editors/{model}/best.pt`: selected tactile-editor checkpoint.
|
| 39 |
+
- `editors/{model}/config.json`, `epochs.jsonl`, and `summary.json`: training
|
| 40 |
+
configuration, validation history, and final physical metrics.
|
| 41 |
+
- `editors/{model}/test_*.npy`: predictions for normal touch, shuffled touch,
|
| 42 |
+
and the zero-touch control.
|
| 43 |
+
- `audit/comparison.json`: Table 1 comparison and physical errors.
|
| 44 |
+
- `code`: the exact cache, training, audit, and tactile-editor implementation
|
| 45 |
+
snapshot used for this release.
|
| 46 |
+
- `SHA256SUMS`: integrity hashes for all other files in this directory.
|
| 47 |
+
|
| 48 |
+
The zero-touch reference gate was explicitly audited: the editor returns the
|
| 49 |
+
base action to numerical precision (maximum action difference at most
|
| 50 |
+
`7.5e-9`).
|
| 51 |
+
|
| 52 |
+
## Minimal loading example
|
| 53 |
+
|
| 54 |
+
```python
|
| 55 |
+
from pathlib import Path
|
| 56 |
+
import numpy as np
|
| 57 |
+
|
| 58 |
+
root = Path("tachin_paired30_cache_20260923")
|
| 59 |
+
target = np.load(root / "cache/shared/test/target_action.npy", mmap_mode="r")
|
| 60 |
+
mask = np.load(root / "cache/shared/test/action_mask.npy", mmap_mode="r")
|
| 61 |
+
smolvla = np.load(root / "cache/bases/smolvla/test/base_action.npy", mmap_mode="r")
|
| 62 |
+
```
|
| 63 |
+
|
| 64 |
+
Use `code/scripts/train_tachin_table1_editor.py` to train an editor from this
|
| 65 |
+
cache. Paths stored inside provenance JSON files are the original absolute run
|
| 66 |
+
paths; use paths relative to this release when reproducing the experiment.
|
| 67 |
+
|
| 68 |
+
## Selected editor results
|
| 69 |
+
|
| 70 |
+
Physical test errors are measured against the Tachin ground truth.
|
| 71 |
+
|
| 72 |
+
| Method | Translation (cm) | Rotation (degree) | Finger MPJPE (cm) | Selected epoch |
|
| 73 |
+
|---|---:|---:|---:|---:|
|
| 74 |
+
| SmolVLA + editor | 1.2413 | 4.6655 | 1.3737 | 10 |
|
| 75 |
+
| VITRA + editor | 1.3520 | 5.5228 | 0.6283 | 9 |
|
| 76 |
+
| T-Rex + editor | 1.2979 | 4.7934 | 1.1978 | 9 |
|
| 77 |
+
| T-WLA W3 + editor | 1.2665 | 4.9293 | 0.4751 | 8 |
|
| 78 |
+
|
| 79 |
+
## Code and environment
|
| 80 |
+
|
| 81 |
+
The source snapshot corresponds to repository revision
|
| 82 |
+
`0a2fec56fad4e2d830403d1371e44dfbd5edc270` on branch
|
| 83 |
+
`tactile_editor_root`, with additional local experiment files included here.
|
| 84 |
+
Package versions are recorded in `environment.txt`.
|
| 85 |
+
|
model/tachin_paired30_cache_20260923/SHA256SUMS
ADDED
|
@@ -0,0 +1,264 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
011e867c049464f7176fccd0d1a4cf550d90ac0e2bd1406548559e1eecb079ac ./README.md
|
| 2 |
+
234fac67e7a7bb128cefbdb56c7c78d4f0852b329202fd8b01d56d015ff5f1c5 ./audit/cache_audit.json
|
| 3 |
+
26adbd78cb19c8d5deec76fb03bc8c20659bfadf05d7addba12f7e813667cc5d ./audit/comparison.json
|
| 4 |
+
2080313884c15626cefee53153b15a3ba3d316f6d31e04c9b82e0eb4d30bf744 ./cache/bases/smolvla/test/base_action.npy
|
| 5 |
+
8c17503a945e74df1b6073faabcd7171bc9e62188a2f8bb8888c85e86add57d5 ./cache/bases/smolvla/test/complete.json
|
| 6 |
+
b7a466bff507433809b0252ec67be686f9cc5779518d603e80103fcd56dbf6ea ./cache/bases/smolvla/test/row_complete.npy
|
| 7 |
+
350ffc92743684d1b17d55fb0154030260f1e66941e2ac3064d65d9320b064e7 ./cache/bases/smolvla/test/smoke_base_action.npy
|
| 8 |
+
9c5a4540fcd1e0dc307098950bc9aa6baabfd7f809ae253e416d529379197939 ./cache/bases/smolvla/test/smoke_complete.json
|
| 9 |
+
ac537e78759740605ab6c13c1755caf6d517d19b47b4b20cf679e5fed978ed67 ./cache/bases/smolvla/test/smoke_row_complete.npy
|
| 10 |
+
582829bcd03feb04814e06a19e371e6b5be4ed19ab811cb9bb2131c5b371d4d6 ./cache/bases/smolvla/train/base_action.npy
|
| 11 |
+
8810e5cd488786b89f22e9e4c46f6dac670cd0f0c1a485e7a8a425f193731a32 ./cache/bases/smolvla/train/complete.json
|
| 12 |
+
986043163cdc5d322abd94b3f38aa38b27bbe0cccdaf3388cd912d63d2aa17ff ./cache/bases/smolvla/train/row_complete.npy
|
| 13 |
+
03a701b74f9e23c0a272768ac4d04d33257c53623a5c305226fcca65f17f2f66 ./cache/bases/smolvla/train/smoke_base_action.npy
|
| 14 |
+
5dbf6b8c835c9a38deb11bfcf8a3b9782ee83c49abad4791139c2ca28950cb66 ./cache/bases/smolvla/train/smoke_complete.json
|
| 15 |
+
ac537e78759740605ab6c13c1755caf6d517d19b47b4b20cf679e5fed978ed67 ./cache/bases/smolvla/train/smoke_row_complete.npy
|
| 16 |
+
5007d2be755f0ae19eb02955b59d4f2c779933c4e2e990d1c57cfbbaced2e449 ./cache/bases/smolvla/val/base_action.npy
|
| 17 |
+
c5ecbada9581dfe6ae627bea5ade115b9dd6bda528d060bb7631f23885a3a1a1 ./cache/bases/smolvla/val/complete.json
|
| 18 |
+
ec8bbb1bd9074ae02818c9b23996b2bfd206ff8d6088db2b1f529c1c15ef845f ./cache/bases/smolvla/val/row_complete.npy
|
| 19 |
+
b30bbf9880451281e79f165c517f9dee25a100426be3f47e4c4361cf211b34fb ./cache/bases/smolvla/val/smoke_base_action.npy
|
| 20 |
+
22309cb83f233a9b83f557ef0a727b218a61183a341359090bbcff0a30001eb1 ./cache/bases/smolvla/val/smoke_complete.json
|
| 21 |
+
ac537e78759740605ab6c13c1755caf6d517d19b47b4b20cf679e5fed978ed67 ./cache/bases/smolvla/val/smoke_row_complete.npy
|
| 22 |
+
1ddfe3b9ea6bbab2b8feda241005e7135fc054c6c0c13d3e42819cc5fb18e0e8 ./cache/bases/trex/test/base_action.npy
|
| 23 |
+
3390c158bb4cbd8fe1bddb6c95e13699be7795906bf687e2fe75ff6ed7a0f07c ./cache/bases/trex/test/complete.json
|
| 24 |
+
300e7137567e46dc1ed50689269205253a37e76cdbc265c871dc91d832b2ab37 ./cache/bases/trex/test/complete_shard_00_of_04.json
|
| 25 |
+
217e4d0e35d39fbdd533eee84e30f28fcb44967891e9a4f490ebe0da4855066c ./cache/bases/trex/test/complete_shard_01_of_04.json
|
| 26 |
+
6d8c718507bc7e819b55a8e92ac6925271df334411905f73dbdf081b78e48a16 ./cache/bases/trex/test/complete_shard_02_of_04.json
|
| 27 |
+
dc1998a3c1057f24a833ae2cd8a76f9f7ad62f05e7bf605c698f269054cdba80 ./cache/bases/trex/test/complete_shard_03_of_04.json
|
| 28 |
+
b7a466bff507433809b0252ec67be686f9cc5779518d603e80103fcd56dbf6ea ./cache/bases/trex/test/row_complete.npy
|
| 29 |
+
10ecb5687069cc23a158fee0a15967511bfca2b6090a075113f2ad4b1ef4b11a ./cache/bases/trex/test/smoke_base_action.npy
|
| 30 |
+
582afc5b9818a720e974ab45340c7ebd5259b01aa2bd14309c73d801b4426bd0 ./cache/bases/trex/test/smoke_complete.json
|
| 31 |
+
d13bab448a58cfff3fb7336525998915b7cde741811292a048f4f925e89edb8f ./cache/bases/trex/test/smoke_row_complete.npy
|
| 32 |
+
a213b36f4483861a589ccb946d39d73c2189f182c83050372971c3cc762d64d2 ./cache/bases/trex/train/base_action.npy
|
| 33 |
+
a7a8408ee35d00c551e56fe1aa14a45f106345844bf76fa2d95d31e384b5dd5a ./cache/bases/trex/train/complete.json
|
| 34 |
+
a7495d9e8a981f6ebdc3e5654e832fe7fe758afa165df078cd18b95edab57683 ./cache/bases/trex/train/complete_shard_00_of_04.json
|
| 35 |
+
4a3703a7cbf8c5e721e8522a68be126e5d3249181783ef044d0cba775f3edb5b ./cache/bases/trex/train/complete_shard_01_of_04.json
|
| 36 |
+
d7d44f9b1fc631a66437afdaabb1dc27fea63169b231191706f23ca5f3d7b341 ./cache/bases/trex/train/complete_shard_02_of_04.json
|
| 37 |
+
72a62a216f146b188280bad3a4749a3bed7b34687d13903c8d96a53279f40543 ./cache/bases/trex/train/complete_shard_03_of_04.json
|
| 38 |
+
986043163cdc5d322abd94b3f38aa38b27bbe0cccdaf3388cd912d63d2aa17ff ./cache/bases/trex/train/row_complete.npy
|
| 39 |
+
ebecfd58f31a466878a9331e457f5f7e2abb17b08d4f1d9e171899d769352073 ./cache/bases/trex/train/smoke_base_action.npy
|
| 40 |
+
4a38c1b9bfc50285cea78c59ab851d948b5348b90375cac67f8dde63ff5089cf ./cache/bases/trex/train/smoke_complete.json
|
| 41 |
+
d13bab448a58cfff3fb7336525998915b7cde741811292a048f4f925e89edb8f ./cache/bases/trex/train/smoke_row_complete.npy
|
| 42 |
+
7b87c9ca444e112e020ce21606e18ef20214e39f50cd09df33bdfb1e97d15489 ./cache/bases/trex/val/base_action.npy
|
| 43 |
+
90431ef3b9199120e450613eefa761a5534e23973fb201070b0cb32cfa184fa3 ./cache/bases/trex/val/complete.json
|
| 44 |
+
b9ef7a09fb539886dd9675419cecbff93d9be271aa11d34b8665f80381879348 ./cache/bases/trex/val/complete_shard_00_of_04.json
|
| 45 |
+
e9c585ab970ba79b1eb6d6af6efac178c017305f8457d7380fb5733ae83840d8 ./cache/bases/trex/val/complete_shard_01_of_04.json
|
| 46 |
+
30232b5ad4050127d1f3c27a157ddd94d477d8be83def05eccd7ca89e2eddec1 ./cache/bases/trex/val/complete_shard_02_of_04.json
|
| 47 |
+
6fa4df77f8ccb4f0e52cdd684a6f08fadf13db1975b6ee08d3aa52e3b4ae4cce ./cache/bases/trex/val/complete_shard_03_of_04.json
|
| 48 |
+
ec8bbb1bd9074ae02818c9b23996b2bfd206ff8d6088db2b1f529c1c15ef845f ./cache/bases/trex/val/row_complete.npy
|
| 49 |
+
f71427f7b16dca5d9009dd7cbf51be3a6a994e7834f5c4dafd65704391ff6f30 ./cache/bases/trex/val/smoke_base_action.npy
|
| 50 |
+
38b66d9746aae67a7d68370f7ea0836120d654da010ff057db4553d3b5c579c0 ./cache/bases/trex/val/smoke_complete.json
|
| 51 |
+
d13bab448a58cfff3fb7336525998915b7cde741811292a048f4f925e89edb8f ./cache/bases/trex/val/smoke_row_complete.npy
|
| 52 |
+
3fa7d31355427bba1eb38c2e3fe57a16952f00e6b25f4722178b042df920e9ba ./cache/bases/twla/test/shard_00/base_action.npy
|
| 53 |
+
88d4a91bad54d4753300c74aabd6d563abc8b4a35619dc82483c6d9245a06fa4 ./cache/bases/twla/test/shard_00/clean_latent.npy
|
| 54 |
+
52ca86fa559c3a4149eb65adbd8160b6b4fda5a1ad8312de0d7b32708268b67c ./cache/bases/twla/test/shard_00/complete.json
|
| 55 |
+
db7c81a10ee497c8575d94ba9f232a755defbf9e41ca6b734096d5d7ac29d103 ./cache/bases/twla/test/shard_00/final_memory.npy
|
| 56 |
+
fc6e0a84d62bbb9d054a428a019f94a89bb30aff8ba006bad254e98a4c40af14 ./cache/bases/twla/test/shard_00/future_roi.npy
|
| 57 |
+
dbbde1b30e9708a9d4218269585b2c0ec191753945a553ea9488c524b93a1cf3 ./cache/bases/twla/test/shard_00/image_tokens.npy
|
| 58 |
+
a8a06cb8e9651cbdd27ce6752e8cc5ab4327db6c0673c176cfa3054e2bb29f6a ./cache/bases/twla/test/shard_00/query_index.npy
|
| 59 |
+
9ad5cfa7086814e276e26484795902a41c83b13bf7c60d2dfd2cb97629ef8610 ./cache/bases/twla/test/shard_00/rgb_valid.npy
|
| 60 |
+
6a87724b86d8d5ed460a5a8be16d5334a351effdef32f8281d19455f17ae0a41 ./cache/bases/twla/test/shard_00/row_complete.npy
|
| 61 |
+
581d0dc8abcac0a070ccc451d60470d3c54227bf9196c8ab8b6b38118e6ca11a ./cache/bases/twla/test/shard_01/base_action.npy
|
| 62 |
+
47a16c5a2dab718bef42c29038b3dcd27c694e6e4443365f5f9ad4783dc02177 ./cache/bases/twla/test/shard_01/clean_latent.npy
|
| 63 |
+
a06e37eaa3f26f9c8499a37c49fd669573f2c769e8cc1f749109c79524863214 ./cache/bases/twla/test/shard_01/complete.json
|
| 64 |
+
290d4fc65379fd8927c183ce67d7604f611067b68452154b98c5722b7acde94f ./cache/bases/twla/test/shard_01/final_memory.npy
|
| 65 |
+
c051b0e49b771a8c7571018e625321c194f0e5e7bd012efe89f5d5cb0f537c1b ./cache/bases/twla/test/shard_01/future_roi.npy
|
| 66 |
+
81582ec12020a0d0927c0c007d970e35aeb8c397a4d214c794b7ae459e90bb69 ./cache/bases/twla/test/shard_01/image_tokens.npy
|
| 67 |
+
658259bd8faeaed759725fcc1bc44cb87192b0727a0e4cfb59456fe78d8c08c0 ./cache/bases/twla/test/shard_01/query_index.npy
|
| 68 |
+
b7cd2b4647e6edfa390e89a0dad83c9472386b161c0fbe725bf70a511c35b378 ./cache/bases/twla/test/shard_01/rgb_valid.npy
|
| 69 |
+
f0a9c6d971f34c731260e6c33f6cfa1443f559950087b1a2b3ac786d5f70ee38 ./cache/bases/twla/test/shard_01/row_complete.npy
|
| 70 |
+
3a6f6993f02aa0451068c65600d8457014406e0bcc3d8dd33c36d53bb6f240c3 ./cache/bases/twla/train/shard_00/base_action.npy
|
| 71 |
+
2615630dc6fccf733397fc85f3b1b38e69ddb93b565a9c03447f1f2081575106 ./cache/bases/twla/train/shard_00/clean_latent.npy
|
| 72 |
+
66915d828f4ba10e14ddcb61223b84da4a5629b397cfc012e1f4e7aed567e478 ./cache/bases/twla/train/shard_00/complete.json
|
| 73 |
+
4ee9df3923fcf9c9b35dac3fd237a7723e9677d35eeef3ff2a6f2f35eea09844 ./cache/bases/twla/train/shard_00/final_memory.npy
|
| 74 |
+
6457be467bc9a04a96f1fe4f6a373f09e0d5b02b5a64dd31863ba89c52735e78 ./cache/bases/twla/train/shard_00/future_roi.npy
|
| 75 |
+
accf30aad2c0dd61bcf85bf40d9c6881ec11b01413c1ee678583d6a19afed418 ./cache/bases/twla/train/shard_00/image_tokens.npy
|
| 76 |
+
f1fe407378b7cbc941e1ff7199ad403ad0919ac333c4746804b158425304bc31 ./cache/bases/twla/train/shard_00/query_index.npy
|
| 77 |
+
8f35c351e5a67a000d3f55071ef6f753d7285567703f2c2ac92520122406ab57 ./cache/bases/twla/train/shard_00/rgb_valid.npy
|
| 78 |
+
7f60693c7b55eab0c4c1d1449c3f4ef7f695023c20b975a00cef6e5288a8ade7 ./cache/bases/twla/train/shard_00/row_complete.npy
|
| 79 |
+
cd39bb8d1fa62ebc15b669c7d2cec56b8447259df6d2f4d632eddc321f0389bf ./cache/bases/twla/train/shard_01/base_action.npy
|
| 80 |
+
7b4fe8cc0b0c74d4b9ae2529f4dab5afb418b9e7dd016947017bce5794932cf1 ./cache/bases/twla/train/shard_01/clean_latent.npy
|
| 81 |
+
4b1b279c01fa4deb577f1a54c877ed588faf325192e05b03669b116f0a94e9fa ./cache/bases/twla/train/shard_01/complete.json
|
| 82 |
+
49e9aa11e02ba8023431ffb763e8bb567f253f991dddd7774fef587a1a078818 ./cache/bases/twla/train/shard_01/final_memory.npy
|
| 83 |
+
fd76cfeabaa2d773a9e2d10ce18ad1520e5d384dd3cdfae9417af4b32d567d1c ./cache/bases/twla/train/shard_01/future_roi.npy
|
| 84 |
+
271ae4ab3219cbb0944db0df693c7a1b8694b7e171f33c6daaa43a641d9756bc ./cache/bases/twla/train/shard_01/image_tokens.npy
|
| 85 |
+
62a3a083607f759477828c3f3d958b347578ca48fc95fda7d446f2e97dcfe49a ./cache/bases/twla/train/shard_01/query_index.npy
|
| 86 |
+
25953dbc165e8f60826eb297f145a41ad23e46be5de98e5554a8c84bdb3b1f67 ./cache/bases/twla/train/shard_01/rgb_valid.npy
|
| 87 |
+
7f60693c7b55eab0c4c1d1449c3f4ef7f695023c20b975a00cef6e5288a8ade7 ./cache/bases/twla/train/shard_01/row_complete.npy
|
| 88 |
+
471795a441696ce123880aa10c01536a9088656daa64df25c924dbbcd04cf43b ./cache/bases/twla/val/shard_00/base_action.npy
|
| 89 |
+
7a58584c88c9dcc87481b7d386f4a618ca7cf17c476afda014aba57a66060fcb ./cache/bases/twla/val/shard_00/clean_latent.npy
|
| 90 |
+
275cad954848f4edeeb2b1df8f5c9122351dea58b62388761c5e9cc592afae89 ./cache/bases/twla/val/shard_00/complete.json
|
| 91 |
+
c3ea8878477559f7608596a544736c512a7189bcbc11ddc8bf7b9ed1a09ca37e ./cache/bases/twla/val/shard_00/final_memory.npy
|
| 92 |
+
de2aa3a0dc1d87729199491eb25b433b56424398492720f28e7eb37bc198303f ./cache/bases/twla/val/shard_00/future_roi.npy
|
| 93 |
+
a5ef36109052b4f13fd12c3c766b9b0979b76a897d84b7afeb1f8fd19a9fabbb ./cache/bases/twla/val/shard_00/image_tokens.npy
|
| 94 |
+
f24f2f19907d8c0f41ab9b48dd2878a9621ac86c1726a6994260039629b11467 ./cache/bases/twla/val/shard_00/query_index.npy
|
| 95 |
+
9f4c0a48e103b67f33de2c4861ac6b54aca17029049c3f343b4c443263c12933 ./cache/bases/twla/val/shard_00/rgb_valid.npy
|
| 96 |
+
bfd4644c8144920f31eff79f61596e2be33fa5fa79059f1b8812ab88b2acf984 ./cache/bases/twla/val/shard_00/row_complete.npy
|
| 97 |
+
acc81026b51aa6a34210357c083639c930b3e1d1f631124353c6f35daf588227 ./cache/bases/twla/val/shard_01/base_action.npy
|
| 98 |
+
453f7aa7fc809e569dd52c53e21a9bb862246d967f36ce94efaa5159405a029f ./cache/bases/twla/val/shard_01/clean_latent.npy
|
| 99 |
+
9ede76b0c8f75d5fd7ffcfbeb0b6432ac2e7076c5e6d6a4a498b2bbfc7e232c7 ./cache/bases/twla/val/shard_01/complete.json
|
| 100 |
+
3b3042e33ad920bc9a410e5b88517b3c90fc6d8fe768236fddd8511b4a2c84dc ./cache/bases/twla/val/shard_01/final_memory.npy
|
| 101 |
+
46ef458b9394ea1d79ee5c02e2f2f9ce401d0a45765395d232ff47f38c89c0c4 ./cache/bases/twla/val/shard_01/future_roi.npy
|
| 102 |
+
490e8402141835a26f4b1c2d4ab415ec59d56644d02c3b0ba8eafca40c8617fa ./cache/bases/twla/val/shard_01/image_tokens.npy
|
| 103 |
+
51514a016dd1890ae289f77da43997114fe68631a706907309388fd9e090a5b3 ./cache/bases/twla/val/shard_01/query_index.npy
|
| 104 |
+
9f4c0a48e103b67f33de2c4861ac6b54aca17029049c3f343b4c443263c12933 ./cache/bases/twla/val/shard_01/rgb_valid.npy
|
| 105 |
+
bfd4644c8144920f31eff79f61596e2be33fa5fa79059f1b8812ab88b2acf984 ./cache/bases/twla/val/shard_01/row_complete.npy
|
| 106 |
+
a95f1ab323df8f5635521757f71833cc2110411e6a91f14297e57fb0c0808ef4 ./cache/bases/vitra/test/base_action.npy
|
| 107 |
+
ba9727c21ade6329c2bc6e7b741146aebdb3fbf370e018cee876f1d97739297c ./cache/bases/vitra/test/complete.json
|
| 108 |
+
b7a466bff507433809b0252ec67be686f9cc5779518d603e80103fcd56dbf6ea ./cache/bases/vitra/test/row_complete.npy
|
| 109 |
+
9338306750dc7822ced14d17e0c66377259d4d58b93f2fff0e6d754be869588f ./cache/bases/vitra/test/smoke_base_action.npy
|
| 110 |
+
99b9ee6097d40303565c4297e76385e52b8b2ff8b91a92557be3a9ecf9cfec47 ./cache/bases/vitra/test/smoke_complete.json
|
| 111 |
+
ac537e78759740605ab6c13c1755caf6d517d19b47b4b20cf679e5fed978ed67 ./cache/bases/vitra/test/smoke_row_complete.npy
|
| 112 |
+
b7d31839721395353da069e46163e92b6ed2f4bfabc582df84c69db3971f0cc9 ./cache/bases/vitra/train/base_action.npy
|
| 113 |
+
36416b564cd2115e718bb1bac066897a32089104056c0307f331ececf1f0253f ./cache/bases/vitra/train/complete.json
|
| 114 |
+
986043163cdc5d322abd94b3f38aa38b27bbe0cccdaf3388cd912d63d2aa17ff ./cache/bases/vitra/train/row_complete.npy
|
| 115 |
+
137cdbe6533a87127f56cf32cd748ac31f79ebdd338743b53a810c757c00dd58 ./cache/bases/vitra/train/smoke_base_action.npy
|
| 116 |
+
a871055da5d32e7f863f303dea8b2de5765f8ede37ae34a01d6298a444bfe69e ./cache/bases/vitra/train/smoke_complete.json
|
| 117 |
+
ac537e78759740605ab6c13c1755caf6d517d19b47b4b20cf679e5fed978ed67 ./cache/bases/vitra/train/smoke_row_complete.npy
|
| 118 |
+
1ded4e5266fbb534454c91d43ca4ca716841a84d8f3296e783e5ccf16301a455 ./cache/bases/vitra/val/base_action.npy
|
| 119 |
+
087760382ecfc13b152e7bb42ca2b3321fee4a45aa5dc71ce0bf936595e24e76 ./cache/bases/vitra/val/complete.json
|
| 120 |
+
ec8bbb1bd9074ae02818c9b23996b2bfd206ff8d6088db2b1f529c1c15ef845f ./cache/bases/vitra/val/row_complete.npy
|
| 121 |
+
5a0269ef27573f876a1ee35ceff21068249c1076d106e12f598cb40d248623e6 ./cache/bases/vitra/val/smoke_base_action.npy
|
| 122 |
+
0a3c55d318db2ca6a24738c83560e97290b9de204eaf7e299dfa82a0a16a8e7f ./cache/bases/vitra/val/smoke_complete.json
|
| 123 |
+
ac537e78759740605ab6c13c1755caf6d517d19b47b4b20cf679e5fed978ed67 ./cache/bases/vitra/val/smoke_row_complete.npy
|
| 124 |
+
cd5818dc9360a1d5aead430c597c06202e5cb607d66918122b9107bed9d1d8e4 ./cache/complete.json
|
| 125 |
+
072e87f8ca325e3f1868a93aea53dc69478d588293fdd092efd0e5f06c38635f ./cache/shared/test/action_mask.npy
|
| 126 |
+
998b06226830defdd6769987a929fce9a3c0e682f6762d8738f9c8a30733d1af ./cache/shared/test/beta.npy
|
| 127 |
+
8c7adad5564c3cfa43472f65fbca41b1497f943d0dea93a3fe2d4a4783fe46fa ./cache/shared/test/complete.json
|
| 128 |
+
a1082dff98c0c90fdad594d35eb07be034d561f4bcddf9b6a889d6bd12d3f6e1 ./cache/shared/test/future_extra_pressure.npy
|
| 129 |
+
cd418be7c9941c85e8d5606f0a15c4d245b0d454c9a98adfcd207e9ef75ccf49 ./cache/shared/test/future_extra_pressure_mask.npy
|
| 130 |
+
e276a2a27c0685a7c633c320e3fce944004d3f9e33440c031d6b1ab5389cbc02 ./cache/shared/test/future_extra_valid.npy
|
| 131 |
+
9abe6b8fd405f60d967dc532603f266228c8730e6e8b520a2e6a0033801df486 ./cache/shared/test/future_pressure.npy
|
| 132 |
+
926e708b285cc8b6e94d73a126a8b714abcbf641a48aefe6a3cdd2ff31ed46ee ./cache/shared/test/future_pressure_mask.npy
|
| 133 |
+
da672a934da52c9e76fe58614712fff4d695dbb1de27958bac53950de40055f6 ./cache/shared/test/future_valid.npy
|
| 134 |
+
57e56a41ca45b82f53d0e2333c80d98d5413edf27d9f3088e57017f4cf25191d ./cache/shared/test/identity.npz
|
| 135 |
+
455055bc47f3b7acf70a023d0bbaa60446b06dedb8a184005b12809edf1dcde4 ./cache/shared/test/pressure.npy
|
| 136 |
+
13937516d2348601e25174ca47fd9545a0e0cd7739c9be60a24ddb766684d7ed ./cache/shared/test/pressure_mask.npy
|
| 137 |
+
328e71ba351a243c5bcd6009b19d7c0aa7c2f32753738558e276d1c2c46c272f ./cache/shared/test/sample_valid.npy
|
| 138 |
+
b7a8a959c5933cef6bd2cbadc5bf0d2feca69bb46c9a6e2cd1915ca73054ceea ./cache/shared/test/target_action.npy
|
| 139 |
+
7e684b5c919fa1a1c1879e054917f61140ce9133b9f4c45b661556af6ee40ac6 ./cache/shared/train/action_mask.npy
|
| 140 |
+
b0ad9e9a421c1890710298662d54ce21293f5f697eaec875ac4f2a7c77b7b31a ./cache/shared/train/beta.npy
|
| 141 |
+
34c4c29d1624479855ddf6bda36a617f66bfde88bdb31df2378d9c56c0cd2053 ./cache/shared/train/complete.json
|
| 142 |
+
7daa9135b33fe209efc890a822574bb46d289ba73d30750ec67f8097c82244a2 ./cache/shared/train/future_extra_pressure.npy
|
| 143 |
+
ebc23669e79bbe51d672d30e510f7b59ee7e9b0922d565a47cc4c2a0bdbf35dd ./cache/shared/train/future_extra_pressure_mask.npy
|
| 144 |
+
9efea0fb8bba7bca3c7921f07ff55f43d4edee264791885800494c1bbe8b1492 ./cache/shared/train/future_extra_valid.npy
|
| 145 |
+
90a63fe771ce15b192171ec773507205615c8962a94cc39977182b793ab591be ./cache/shared/train/future_pressure.npy
|
| 146 |
+
7eed9677354f55d0afd92107ec8f05aca3fb059db3db3b56bbe143d17f452923 ./cache/shared/train/future_pressure_mask.npy
|
| 147 |
+
0cbf19384a4ffcafed6dd87868e167512ba59ab64652d08fabe20062f110e0ab ./cache/shared/train/future_valid.npy
|
| 148 |
+
6062ae9d1969eb09cbbf378963f02376d7d5525db04be06970ebf3815ef8cf56 ./cache/shared/train/identity.npz
|
| 149 |
+
1534ad0f7e0aaabc8a871317618aa0ab7c62be1fb88bae480280a9ac046adbe0 ./cache/shared/train/pressure.npy
|
| 150 |
+
06bfaedd2e75f9ad1f8322ebd2402b2652ef0a8e0e6c42a434d8d740b0a3bd3f ./cache/shared/train/pressure_mask.npy
|
| 151 |
+
ce00e301736362e1d1efa0c182685d1e77d431c763200c6f7e5e34bee66dd97a ./cache/shared/train/sample_valid.npy
|
| 152 |
+
42a48f1b21639ccd00f9bc49553440ee220f64beb2285f8db4456be8d0c32693 ./cache/shared/train/target_action.npy
|
| 153 |
+
8c263711f368b6475605f49198a2b6a5ba9a94f615d5cd27bb5a600348a2669b ./cache/shared/val/action_mask.npy
|
| 154 |
+
7e15197f5c45129ce8f30d0c901eb339126c5237b63db24f441347f751f0dc08 ./cache/shared/val/beta.npy
|
| 155 |
+
339b74ea7926aec4ef516246fdf671fbb33bda4f9a8bc2d0a91f23fbbd988006 ./cache/shared/val/complete.json
|
| 156 |
+
80c9029f6be000061b59527d2198c98549bdef1a56e4e1115363860c59a278f4 ./cache/shared/val/future_extra_pressure.npy
|
| 157 |
+
fd608aa9533c5ea16596d0d80ebe5efdedeba2edf98b8cdd98c0b0df9cf820e3 ./cache/shared/val/future_extra_pressure_mask.npy
|
| 158 |
+
c6a671eead7588e91177cf5ac9e90e61c799c6dd21cb7e231d5251a3c224dd50 ./cache/shared/val/future_extra_valid.npy
|
| 159 |
+
050d008fafbbc7a20f064eac31b727e12af4ddd3ea8631b8cc82398bcaece11d ./cache/shared/val/future_pressure.npy
|
| 160 |
+
6056c81a8603fb3edef9ca066cbc9dd7896d132b8658c599759586c28d782f5a ./cache/shared/val/future_pressure_mask.npy
|
| 161 |
+
44a1aeb5bb4c8f72868cb6663b8e0d4c0ffc16d90a45f2106744b3b2d9023067 ./cache/shared/val/future_valid.npy
|
| 162 |
+
3dab5808ba13c98cd344ac7b94d3938feef570570b4b4788501caea9083792b5 ./cache/shared/val/identity.npz
|
| 163 |
+
32bf4eed1d0d85cfdc0fae69ea5557813407a155ca2dead87f42d084812478ee ./cache/shared/val/pressure.npy
|
| 164 |
+
9dc4c55be149a56a8e97c9491d0d96212bfc8af83d43ab2278761b9a31804800 ./cache/shared/val/pressure_mask.npy
|
| 165 |
+
549f1b76cddf05bdbc7ec1799585526e74cb3f9d48da243b095b2fc8b2e6bd49 ./cache/shared/val/sample_valid.npy
|
| 166 |
+
eb575e92d2ed75b4e9068d21315cac627d6e404af74291f8a570bd95fef54b16 ./cache/shared/val/target_action.npy
|
| 167 |
+
7441a432cad24f900a8d47c7cbbfe559de716c53eb82330ef82edfcd41165e5b ./cache/statistics.npz
|
| 168 |
+
8117bc566c3728c63b5eeb08bbb06772ae83e8629e7e9c8be246a2516836d694 ./code/scripts/audit_tachin_paired30.py
|
| 169 |
+
726e38f8c64cd119653edb4863659742277cefcc8c574a73aae386184ab74334 ./code/scripts/build_tachin_paired30_cache.py
|
| 170 |
+
9c10f52ac1adec1dd488dff2f3fd992200379431b7f70606cfdf6399765245ed ./code/scripts/cache_tachin_adapted_base.py
|
| 171 |
+
f89146ac7f47157deea98e6dd713093f5c80d25af811a8581f553e593de9e7ea ./code/scripts/cache_tachin_twla_w3.py
|
| 172 |
+
6cd5ea561f79c36ec34407c77fce721fe3358c8d01bc596fed85b057a4af9ba3 ./code/scripts/summarize_tachin_paired30.py
|
| 173 |
+
75e6cb8d5ee3117bbf000e28595f69fa9409efdd7e307afda51488e9c2127f0a ./code/scripts/train_tachin_table1_editor.py
|
| 174 |
+
63d688862a2f82dcf06c182947cafc00917698c3c5d551372f956860a534c46b ./code/src/tactile_editor/__init__.py
|
| 175 |
+
9259591d593df5d6d661c7afccc54886a5b7ef2f9493c2ce79ae3d19e304bc30 ./code/src/tactile_editor/_baselines/__init__.py
|
| 176 |
+
e5a0703829318d9704109d11ad068e8fa6fd83a0899ba2eaa86c2d0b2ecf8fe6 ./code/src/tactile_editor/_baselines/provenance.json
|
| 177 |
+
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 ./code/src/tactile_editor/_baselines/smolvla_official/__init__.py
|
| 178 |
+
ca4af48d152030f4741068308f9b0faac63af18178eefd8c9cbf2efe2b776f34 ./code/src/tactile_editor/_baselines/smolvla_official/config.py
|
| 179 |
+
8ec960f681de59fc47bcc9dc0a048b5f6f095ee83259e4ced6b85d2196c414fd ./code/src/tactile_editor/_baselines/smolvla_official/flow.py
|
| 180 |
+
f8f26dc7af38af10db0d012d527a53c77493a034d96c0fda4e7462d2f6af0164 ./code/src/tactile_editor/_baselines/smolvla_official/group_loss.py
|
| 181 |
+
c7f6dac2430db912f277fd8442011269207cd3e163a88705eceb55436443b733 ./code/src/tactile_editor/_baselines/smolvla_official/model.py
|
| 182 |
+
8b17d09c5ab9b68e324e1b4f5011ab21e8d66165d8c955f3d36c19469229619e ./code/src/tactile_editor/_baselines/smolvla_official/smolvlm_with_expert.py
|
| 183 |
+
689fea725369aba4f7bb5eb72112eb7f96911061bbde225113793626d7b687b2 ./code/src/tactile_editor/_baselines/smolvla_official/vla_utils.py
|
| 184 |
+
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 ./code/src/tactile_editor/_baselines/vitra_original/__init__.py
|
| 185 |
+
306ce5d342acbe665c9160771b4c3711463a70598fddbc39a41df5362d7bdbc7 ./code/src/tactile_editor/_baselines/vitra_original/config.py
|
| 186 |
+
43f8e9932aac6f98258bdf7902815b06c37a9e6f619f2733ce242d0bdda396ee ./code/src/tactile_editor/_baselines/vitra_original/diffusion.py
|
| 187 |
+
7448464d79b9bbda800688798ba825e1864f9240502d2a5f6848e5d650074b73 ./code/src/tactile_editor/_baselines/vitra_original/dit.py
|
| 188 |
+
bb021ee4c9d8f38db633f1f60fc8f9ffc6c4b963f442850233f794c42a79524a ./code/src/tactile_editor/_baselines/vitra_original/group_loss.py
|
| 189 |
+
edaaf0ce60a97459594f034a53a8edf9b1b07d2354cecf829ac74e957848bdc7 ./code/src/tactile_editor/_baselines/vitra_original/model.py
|
| 190 |
+
b0816ec6cd3f66e61bbd5ebd77b393a794a847fa0e801ca6a174b82ba6876fc7 ./code/src/tactile_editor/_baselines/vitra_original/prompt.py
|
| 191 |
+
f8e4f6c753f660201398a544d2b8f32bb94b573573acb21037221a341cb68807 ./code/src/tactile_editor/_baselines/vitra_original/vlm.py
|
| 192 |
+
5cfaab144e97a49fc928cc04ab5fd77dbaa380d22bed38ec98f007feb7babe28 ./code/src/tactile_editor/action_layout.py
|
| 193 |
+
2179bb8ecba4dc3a74086853c0c61ca1bbd2e2b1021c0054438f6541d4a6a144 ./code/src/tactile_editor/action_objectives.py
|
| 194 |
+
5e09d6a397a5e048cfe201c572fa9f3f7660743c7cba3ce6b7ccf0e654ca5430 ./code/src/tactile_editor/action_output.py
|
| 195 |
+
7ef6141988b5e7bb33e44537b4a8a226b2934b9a1e685545ed7f71e90f72a77f ./code/src/tactile_editor/architecture_ablation.py
|
| 196 |
+
a3f3b5ac96322efbbedb496f562e5ba34e39cdc767da3105dfb9bb10f28fd072 ./code/src/tactile_editor/auxiliary_objectives.py
|
| 197 |
+
0ffc136277938b915e770da6c3b27bef08abffd4238b13a3c269208010009a2b ./code/src/tactile_editor/checkpoint_loaders.py
|
| 198 |
+
d441fd06da718276de55811a262db2d5beb684a70ec088d9a2bded5cd0581b7e ./code/src/tactile_editor/config.py
|
| 199 |
+
b15af5c93cd5db8a4888d32e15476854815fb8e1cc729601456c1928d048cfce ./code/src/tactile_editor/contact_filter.py
|
| 200 |
+
9ec5b7e82350fb1167b0ea7894c50e7969af6e072b2a59494bc0943eaae8e4c0 ./code/src/tactile_editor/direct_residual.py
|
| 201 |
+
4725aee12e9f98d942b494325c01dffa40a37b59195f8e929f36949f812ddfb4 ./code/src/tactile_editor/editor_backends.py
|
| 202 |
+
8d0503690a247468fee690fe7bb16b3b6b6272ba709d8cf93f739832fbe89a46 ./code/src/tactile_editor/editor_interface.py
|
| 203 |
+
5b889e99f587e66b4cae09fd13cefe517ba357883cac1e8386d4a0296203a793 ./code/src/tactile_editor/encoder.py
|
| 204 |
+
82d51681d4d3631ce876ee9ee68d12263ec0960f1de8636f02f5013f71c98dcc ./code/src/tactile_editor/frozen_stage1.py
|
| 205 |
+
a5810e83a7b5c70f0630110b0dece8703750902b10973e850a79c2572b13f457 ./code/src/tactile_editor/full_cache.py
|
| 206 |
+
385f2df034fc45f91464507dbf88827518a240d8c2ce5ce1aecdde58ec8d84ec ./code/src/tactile_editor/fusion.py
|
| 207 |
+
988ea09d24e771dd3f6db4b08a78467421cd41ccec18a72916a2186c28c3e517 ./code/src/tactile_editor/human_baselines.py
|
| 208 |
+
b09c0afff1f5fc99c4f104dd8ee998922c639f199ffdb0e4b18cf88bc591f1c5 ./code/src/tactile_editor/informative_data.py
|
| 209 |
+
054f426876e418a9bc243e70512a27bdbfb281aa092f91f4dcd422ef0936d88c ./code/src/tactile_editor/mano_action.py
|
| 210 |
+
b0fedec1b8f6d104ea8cd2551e19d4bc499f3162c256a73589eb4f06a4e433a2 ./code/src/tactile_editor/memoryvla_baseline.py
|
| 211 |
+
dd6f77f9967f66928a02da1ebdf6ee4c36d163164eeda3e95f32431189611e64 ./code/src/tactile_editor/mock.py
|
| 212 |
+
0c8dbb0d1da02d95d4e69b45abe091fbe7556ac8651640e26932ec42628e3ead ./code/src/tactile_editor/model.py
|
| 213 |
+
9e60ec589dc4fc63fe7082a0d9422bda942963e9b033810ec3580f94607ba7bc ./code/src/tactile_editor/model_adapters.py
|
| 214 |
+
6627dd50c1fdcf18b1df11c7387875dc62a91df6412573bb5926d0c2e9d29b30 ./code/src/tactile_editor/native_tachin.py
|
| 215 |
+
14ddc4222f42ef3c03d333e490bfae4bf8049dedd256d08d5fa3172f096b6b05 ./code/src/tactile_editor/offline_data.py
|
| 216 |
+
fb5a907bbe4e24912244e4eec58f3e369201222eb8dff6774a970940416ba777 ./code/src/tactile_editor/paired30_cache.py
|
| 217 |
+
c2c8cd891d297509bbb63fc921d5be1b68af0f3ccaab65d03369ef96c3e03835 ./code/src/tactile_editor/real_data.py
|
| 218 |
+
b2def8c80c9515f6686ff689f5ce5c926d74ca8309d05161cd6b1c315072518d ./code/src/tactile_editor/rgb_world_data.py
|
| 219 |
+
8bd6478f81aaf058b73a2a7cd8ccd646e22c841d53a7479a8d48328be4888040 ./code/src/tactile_editor/rgb_world_editor.py
|
| 220 |
+
e0d979702559b5361d53a1f3df0cb2a3ed999b9496aaea9531280eb17def02e4 ./code/src/tactile_editor/runtime.py
|
| 221 |
+
585edb9909006d293b01f0a0b22499e53b9db9d145b9d754402c5c615192035d ./code/src/tactile_editor/single_stage_world.py
|
| 222 |
+
6e97c4a65f68e701c9dafdd73964843e6b17ed7ec1265e206347c48d97e2da9b ./code/src/tactile_editor/smoke.py
|
| 223 |
+
d262a8c2075d200eb51cbd246bc6f3cec7a1d52b4eb19c1566b93b173b77bce9 ./code/src/tactile_editor/table1_editor_cache.py
|
| 224 |
+
bcf4d1e566a29e6d17fd608d51502bc832b91ee9c23d0a554980723c35829846 ./code/src/tactile_editor/table1_experiment.py
|
| 225 |
+
d74683999995a121e8a31d4da277d554e83caa8cfdb5f39d5a157eed18832590 ./code/src/tactile_editor/table1_protocol.py
|
| 226 |
+
e0bd6df948b0004ad15756f58079aee128d75058e964971b121061019a26adec ./code/src/tactile_editor/touch_study.py
|
| 227 |
+
8d74e8aed5a765d6d5f7b0ab4b77ed02beaa8e460f83bc7429eb1cdc92ba3ebc ./code/src/tactile_editor/trainability.py
|
| 228 |
+
cb7913357ee11ee8c84a739aea3c15fef3bae809c2d57c680a018559b205d95e ./code/src/tactile_editor/training.py
|
| 229 |
+
40e5eb86b8ea6dea91d2cbc0755005ea857751946ed0c28518d960540244dac5 ./code/src/tactile_editor/unified_cache.py
|
| 230 |
+
c10fb673b0af2c6ac3673cb8220facfd9a7fb20b68419fe82270f569866cbe1f ./code/src/tactile_editor/unified_world.py
|
| 231 |
+
a5608efcaee31e8f6a74bea32dbfa088fc2f3dce209a4b1c5d77e3e61a5256cf ./code/src/tactile_editor/visual_dynamics_pilot.py
|
| 232 |
+
e0d895c3c8661b7cde1be24f7a343a9d0da5bd62921d96928c8eb2dcdb317bf3 ./code/src/tactile_editor/world_flow.py
|
| 233 |
+
a9e5d2b77c83b53863f5cde861b0d7641032f5efec5dd31c916daf6af3c98aea ./code/src/tactile_editor/world_regularized_data.py
|
| 234 |
+
058c61c0e36c95a684138394b7275d71b9db66aa3faaac9a5f444014c9a0eb26 ./code/src/tactile_editor/world_residual_probe.py
|
| 235 |
+
30541b63e49b9a68f20dfb65f775d5076a5ebcd1b7cdb30957d43d5b2aace6f6 ./code/tests/test_tachin_native_editor.py
|
| 236 |
+
da3a1e31ad14e9844532228121545af4e1bb3dcae443dae7c4a20f372c893884 ./editors/smolvla/best.pt
|
| 237 |
+
6c6bf15aee4036efcc2009ade5d66e35803e912add518a552e6808c0c9343cfe ./editors/smolvla/config.json
|
| 238 |
+
d7fcd2c60a7cce29603d5afb065c2e127ffa91eb90a7848aed9149d1efb8f913 ./editors/smolvla/epochs.jsonl
|
| 239 |
+
16cd7b468c98d1bb58c813b2688f60852e888b6bd88fd88d5aefb0d220a22964 ./editors/smolvla/summary.json
|
| 240 |
+
933932400d7925a5a54d2f71ed6ab8efd191463c4a8d52750e93d0bddf327bc5 ./editors/smolvla/test_editor_action.npy
|
| 241 |
+
27fd2986caba17c53b0061aaf0cf4b89653e9a88f266bf1973855f2092927fa9 ./editors/smolvla/test_shuffled_touch_action.npy
|
| 242 |
+
daf66f08fa5a2b477c7a54d37df30d2dc14e2938f047ab7eb81bacc8c37d3da1 ./editors/smolvla/test_zero_touch_action.npy
|
| 243 |
+
c39c64ea31d1080c3ff8423bd6621639420dd39a5ff3726e60d9ddcf981caf73 ./editors/trex/best.pt
|
| 244 |
+
9b0e34d73a062c8a2636cec3fa046cfdd788b2bb81b0d85f1642c04b9fc7ea8b ./editors/trex/config.json
|
| 245 |
+
9463a52418c7f3544a7cb92456ba6e6b8f66a1cdb205c46d600e9e3b42772acb ./editors/trex/epochs.jsonl
|
| 246 |
+
d3e07ee3a1d429b1d6d8a1f8115521c67e851b0e16aa6c3cb4ed13891e41b6a0 ./editors/trex/summary.json
|
| 247 |
+
6be42371c333061b410c9c3678a4b0e4802ec473de37557daff79b38acd30868 ./editors/trex/test_editor_action.npy
|
| 248 |
+
fede955cf5dbecec4dbe92dcec1901b052f24f14e409c669c10f07d2b99bde84 ./editors/trex/test_shuffled_touch_action.npy
|
| 249 |
+
80f2ee3d3f08260e783ef560cb3f9211d4f650dd9d0963bc7f18bd257331e38e ./editors/trex/test_zero_touch_action.npy
|
| 250 |
+
3955de4381cfe33223081eea7589dab1a32af99e65e4ba4a392a82102fc5c1b0 ./editors/twla/best.pt
|
| 251 |
+
1300487a3dd85b48efeddef92e4887ad1683c22e619197579ebec3922ad7154d ./editors/twla/config.json
|
| 252 |
+
52d2bad73d91de86898440079a134c91a9907cbb40d9e1c67899c19885fe615d ./editors/twla/epochs.jsonl
|
| 253 |
+
0190786e58f244c0e2d021b1870ec7b6a8992af1e503875da8ad2842ca4fe613 ./editors/twla/summary.json
|
| 254 |
+
409497ac11a9fecb1792c30ccdc8e876ebc9ef73ac776e2526de3f608edcc3cd ./editors/twla/test_editor_action.npy
|
| 255 |
+
5c582cb663f2f6483f9f5ea99a50c815c5c5d05ef5315efe35a768a19bae9ce7 ./editors/twla/test_shuffled_touch_action.npy
|
| 256 |
+
0fb186f3b70c85d3c1d0cdfd93b4ac7940a0d94207498f3b54ac2e6e1a07ef04 ./editors/twla/test_zero_touch_action.npy
|
| 257 |
+
5f9d4bd6c0ea2170e22bed59f4448d2f975e1e6a66b8c4a91616a6f02954bd47 ./editors/vitra/best.pt
|
| 258 |
+
98132855d066b96c4661ca84392aba547eb6e4b233396971ae74a56f5bc14280 ./editors/vitra/config.json
|
| 259 |
+
532ea7340f79863c20306a0cbc374fd9115af545bee0d616e4506c7991c1b5c5 ./editors/vitra/epochs.jsonl
|
| 260 |
+
8689e18fa3bff2bb29903e43e55a17b7fac274f129de4e6efb37784982e074bd ./editors/vitra/summary.json
|
| 261 |
+
9e2724e241b3698b801eff61e6f682de08f22659f394a79b2adeedc5edefaecc ./editors/vitra/test_editor_action.npy
|
| 262 |
+
7cce8d5d3e57304d9728f70d97982160d79df36dd486cb01beffdde958967160 ./editors/vitra/test_shuffled_touch_action.npy
|
| 263 |
+
e8b1afc84e34207f431227985de120463fc801019d8c178f218b93dbec451b29 ./editors/vitra/test_zero_touch_action.npy
|
| 264 |
+
b7a5464413a57d315ad3064bc6ca2c9d8bc24e791962065282fca16de5ef35c7 ./environment.txt
|
model/tachin_paired30_cache_20260923/audit/cache_audit.json
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"cache": "/scratch/hannahyao24/VLA-tactile_paper/artifacts/tachin_table1_paired30_20260923",
|
| 3 |
+
"splits": {
|
| 4 |
+
"train": {
|
| 5 |
+
"plans": 18208,
|
| 6 |
+
"paired_rows": 49744,
|
| 7 |
+
"rows_by_decision_tick": [
|
| 8 |
+
16576,
|
| 9 |
+
16596,
|
| 10 |
+
16572
|
| 11 |
+
],
|
| 12 |
+
"models_share_exact_rows": true,
|
| 13 |
+
"causal_windows_exact": true,
|
| 14 |
+
"decision_relative_w3_targets_exact": true,
|
| 15 |
+
"first_editable_action": [
|
| 16 |
+
0,
|
| 17 |
+
3,
|
| 18 |
+
6
|
| 19 |
+
]
|
| 20 |
+
},
|
| 21 |
+
"val": {
|
| 22 |
+
"plans": 1006,
|
| 23 |
+
"paired_rows": 2655,
|
| 24 |
+
"rows_by_decision_tick": [
|
| 25 |
+
883,
|
| 26 |
+
886,
|
| 27 |
+
886
|
| 28 |
+
],
|
| 29 |
+
"models_share_exact_rows": true,
|
| 30 |
+
"causal_windows_exact": true,
|
| 31 |
+
"decision_relative_w3_targets_exact": true,
|
| 32 |
+
"first_editable_action": [
|
| 33 |
+
0,
|
| 34 |
+
3,
|
| 35 |
+
6
|
| 36 |
+
]
|
| 37 |
+
},
|
| 38 |
+
"test": {
|
| 39 |
+
"plans": 1169,
|
| 40 |
+
"paired_rows": 2433,
|
| 41 |
+
"rows_by_decision_tick": [
|
| 42 |
+
809,
|
| 43 |
+
812,
|
| 44 |
+
812
|
| 45 |
+
],
|
| 46 |
+
"models_share_exact_rows": true,
|
| 47 |
+
"causal_windows_exact": true,
|
| 48 |
+
"decision_relative_w3_targets_exact": true,
|
| 49 |
+
"first_editable_action": [
|
| 50 |
+
0,
|
| 51 |
+
3,
|
| 52 |
+
6
|
| 53 |
+
]
|
| 54 |
+
}
|
| 55 |
+
},
|
| 56 |
+
"status": "PASS"
|
| 57 |
+
}
|
model/tachin_paired30_cache_20260923/audit/comparison.json
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"protocol": "Tachin paired-30",
|
| 3 |
+
"models": {
|
| 4 |
+
"smolvla": {
|
| 5 |
+
"training_budget_epochs": 10,
|
| 6 |
+
"selected_epoch": 10,
|
| 7 |
+
"validation_action_loss": 0.2469381999610283,
|
| 8 |
+
"base_metrics": {
|
| 9 |
+
"translation_cm": 1.3014105013364496,
|
| 10 |
+
"rotation_deg": 4.997658153014678,
|
| 11 |
+
"joint_mpjpe_cm": 1.6248546858030588
|
| 12 |
+
},
|
| 13 |
+
"editor_metrics": {
|
| 14 |
+
"translation_cm": 1.2412666298214436,
|
| 15 |
+
"rotation_deg": 4.665520451312771,
|
| 16 |
+
"joint_mpjpe_cm": 1.3737346600468263
|
| 17 |
+
},
|
| 18 |
+
"shuffled_touch_metrics": {
|
| 19 |
+
"translation_cm": 1.2407394562784146,
|
| 20 |
+
"rotation_deg": 4.663409782922765,
|
| 21 |
+
"joint_mpjpe_cm": 1.3714706658511227
|
| 22 |
+
},
|
| 23 |
+
"zero_touch_metrics": {
|
| 24 |
+
"translation_cm": 1.3014105013364496,
|
| 25 |
+
"rotation_deg": 4.99765815311085,
|
| 26 |
+
"joint_mpjpe_cm": 1.6248546858030588
|
| 27 |
+
},
|
| 28 |
+
"table1_cells": {
|
| 29 |
+
"translation_cm": "0.06 / (4.62%)",
|
| 30 |
+
"rotation_deg": "0.33 / (6.65%)",
|
| 31 |
+
"joint_mpjpe_cm": "0.25 / (15.45%)"
|
| 32 |
+
},
|
| 33 |
+
"zero_touch_max_action_difference": 3.725290298461914e-09,
|
| 34 |
+
"editor_vs_shuffled_action_mae": 0.0003912480897270143
|
| 35 |
+
},
|
| 36 |
+
"vitra": {
|
| 37 |
+
"training_budget_epochs": 10,
|
| 38 |
+
"selected_epoch": 9,
|
| 39 |
+
"validation_action_loss": 0.19477003950295924,
|
| 40 |
+
"base_metrics": {
|
| 41 |
+
"translation_cm": 1.7086415295051836,
|
| 42 |
+
"rotation_deg": 7.852093026854487,
|
| 43 |
+
"joint_mpjpe_cm": 0.6802416275515677
|
| 44 |
+
},
|
| 45 |
+
"editor_metrics": {
|
| 46 |
+
"translation_cm": 1.3520344510315767,
|
| 47 |
+
"rotation_deg": 5.52281293424393,
|
| 48 |
+
"joint_mpjpe_cm": 0.6283003161906104
|
| 49 |
+
},
|
| 50 |
+
"shuffled_touch_metrics": {
|
| 51 |
+
"translation_cm": 1.3519262729830466,
|
| 52 |
+
"rotation_deg": 5.5235926742795565,
|
| 53 |
+
"joint_mpjpe_cm": 0.6271929750119718
|
| 54 |
+
},
|
| 55 |
+
"zero_touch_metrics": {
|
| 56 |
+
"translation_cm": 1.7086415295354782,
|
| 57 |
+
"rotation_deg": 7.852093026936965,
|
| 58 |
+
"joint_mpjpe_cm": 0.6802416272885641
|
| 59 |
+
},
|
| 60 |
+
"table1_cells": {
|
| 61 |
+
"translation_cm": "0.36 / (20.87%)",
|
| 62 |
+
"rotation_deg": "2.33 / (29.66%)",
|
| 63 |
+
"joint_mpjpe_cm": "0.05 / (7.64%)"
|
| 64 |
+
},
|
| 65 |
+
"zero_touch_max_action_difference": 7.450580596923828e-09,
|
| 66 |
+
"editor_vs_shuffled_action_mae": 7.935501344036311e-05
|
| 67 |
+
},
|
| 68 |
+
"trex": {
|
| 69 |
+
"training_budget_epochs": 10,
|
| 70 |
+
"selected_epoch": 9,
|
| 71 |
+
"validation_action_loss": 0.2388624380761845,
|
| 72 |
+
"base_metrics": {
|
| 73 |
+
"translation_cm": 1.5101441290157958,
|
| 74 |
+
"rotation_deg": 5.626478859715439,
|
| 75 |
+
"joint_mpjpe_cm": 1.2290510475647314
|
| 76 |
+
},
|
| 77 |
+
"editor_metrics": {
|
| 78 |
+
"translation_cm": 1.2979139497825751,
|
| 79 |
+
"rotation_deg": 4.793363297812164,
|
| 80 |
+
"joint_mpjpe_cm": 1.1977501825700754
|
| 81 |
+
},
|
| 82 |
+
"shuffled_touch_metrics": {
|
| 83 |
+
"translation_cm": 1.2985424613853025,
|
| 84 |
+
"rotation_deg": 4.801285892773723,
|
| 85 |
+
"joint_mpjpe_cm": 1.197848930770959
|
| 86 |
+
},
|
| 87 |
+
"zero_touch_metrics": {
|
| 88 |
+
"translation_cm": 1.5101441290157958,
|
| 89 |
+
"rotation_deg": 5.626478859702203,
|
| 90 |
+
"joint_mpjpe_cm": 1.2290510475647314
|
| 91 |
+
},
|
| 92 |
+
"table1_cells": {
|
| 93 |
+
"translation_cm": "0.21 / (14.05%)",
|
| 94 |
+
"rotation_deg": "0.83 / (14.81%)",
|
| 95 |
+
"joint_mpjpe_cm": "0.03 / (2.55%)"
|
| 96 |
+
},
|
| 97 |
+
"zero_touch_max_action_difference": 3.725290298461914e-09,
|
| 98 |
+
"editor_vs_shuffled_action_mae": 0.0001029103368637152
|
| 99 |
+
},
|
| 100 |
+
"twla": {
|
| 101 |
+
"training_budget_epochs": 30,
|
| 102 |
+
"selected_epoch": 8,
|
| 103 |
+
"validation_action_loss": 0.1752967903169535,
|
| 104 |
+
"base_metrics": {
|
| 105 |
+
"translation_cm": 1.3520767220769714,
|
| 106 |
+
"rotation_deg": 5.292899332876988,
|
| 107 |
+
"joint_mpjpe_cm": 0.5639765192191974
|
| 108 |
+
},
|
| 109 |
+
"editor_metrics": {
|
| 110 |
+
"translation_cm": 1.2665439394654596,
|
| 111 |
+
"rotation_deg": 4.929279282635632,
|
| 112 |
+
"joint_mpjpe_cm": 0.47511430038172686
|
| 113 |
+
},
|
| 114 |
+
"shuffled_touch_metrics": {
|
| 115 |
+
"translation_cm": 1.2630061238186077,
|
| 116 |
+
"rotation_deg": 4.932078891277113,
|
| 117 |
+
"joint_mpjpe_cm": 0.47602607038290146
|
| 118 |
+
},
|
| 119 |
+
"zero_touch_metrics": {
|
| 120 |
+
"translation_cm": 1.352076721918785,
|
| 121 |
+
"rotation_deg": 5.292899332792065,
|
| 122 |
+
"joint_mpjpe_cm": 0.5639765192191974
|
| 123 |
+
},
|
| 124 |
+
"table1_cells": {
|
| 125 |
+
"translation_cm": "0.09 / (6.33%)",
|
| 126 |
+
"rotation_deg": "0.36 / (6.87%)",
|
| 127 |
+
"joint_mpjpe_cm": "0.09 / (15.76%)"
|
| 128 |
+
},
|
| 129 |
+
"zero_touch_max_action_difference": 3.725290298461914e-09,
|
| 130 |
+
"editor_vs_shuffled_action_mae": 0.0001719161809887737
|
| 131 |
+
}
|
| 132 |
+
},
|
| 133 |
+
"twla_vs_best_competing_editor": {
|
| 134 |
+
"translation_cm": {
|
| 135 |
+
"best_competing_model": "smolvla",
|
| 136 |
+
"best_competing_error": 1.2412666298214436,
|
| 137 |
+
"twla_error": 1.2665439394654596,
|
| 138 |
+
"twla_reduction_percent_vs_best_competitor": -2.0364125673508293
|
| 139 |
+
},
|
| 140 |
+
"rotation_deg": {
|
| 141 |
+
"best_competing_model": "smolvla",
|
| 142 |
+
"best_competing_error": 4.665520451312771,
|
| 143 |
+
"twla_error": 4.929279282635632,
|
| 144 |
+
"twla_reduction_percent_vs_best_competitor": -5.653363522362134
|
| 145 |
+
},
|
| 146 |
+
"joint_mpjpe_cm": {
|
| 147 |
+
"best_competing_model": "vitra",
|
| 148 |
+
"best_competing_error": 0.6283003161906104,
|
| 149 |
+
"twla_error": 0.47511430038172686,
|
| 150 |
+
"twla_reduction_percent_vs_best_competitor": 24.381018417697376
|
| 151 |
+
}
|
| 152 |
+
},
|
| 153 |
+
"cache_audit": "/scratch/hannahyao24/VLA-tactile_paper/runs/tachin_paired30_20260923/cache_audit.json",
|
| 154 |
+
"status": "PASS"
|
| 155 |
+
}
|
model/tachin_paired30_cache_20260923/cache/complete.json
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema": "tachin.table1.editor_cache.paired30.v1",
|
| 3 |
+
"counts": {
|
| 4 |
+
"train": 18208,
|
| 5 |
+
"val": 1006,
|
| 6 |
+
"test": 1169
|
| 7 |
+
},
|
| 8 |
+
"source_cache": "/scratch/hannahyao24/VLA-tactile_paper/artifacts/tachin_table1_twla_adapted_20260922",
|
| 9 |
+
"benchmark": "/scratch/hannahyao24/VLA-tactile_paper/artifacts/datasets/tachin_benchmark_v2_release",
|
| 10 |
+
"adaptation_data": "/scratch/hannahyao24/VLA-tactile_paper/artifacts/datasets/tachin_adaptation_20260922",
|
| 11 |
+
"vision_plan_hz": 10.0,
|
| 12 |
+
"tactile_input_hz": 30.0,
|
| 13 |
+
"editor_decision_hz": 30.0,
|
| 14 |
+
"decision_ticks": [
|
| 15 |
+
0,
|
| 16 |
+
1,
|
| 17 |
+
2
|
| 18 |
+
],
|
| 19 |
+
"action_hz": 80.0,
|
| 20 |
+
"gated": false
|
| 21 |
+
}
|
model/tachin_paired30_cache_20260923/code/scripts/audit_tachin_paired30.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Fail-closed audit of the native Tachin paired-30 cache and row contract."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import argparse
|
| 5 |
+
import json
|
| 6 |
+
import sys
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
|
| 9 |
+
import numpy as np
|
| 10 |
+
import torch
|
| 11 |
+
|
| 12 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 13 |
+
sys.path[:0] = [str(ROOT / "src"), str(ROOT / "scripts")]
|
| 14 |
+
from train_tachin_table1_editor import TachinEditorCache, edit_mask
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def main() -> None:
|
| 18 |
+
parser = argparse.ArgumentParser()
|
| 19 |
+
parser.add_argument("--cache", type=Path, required=True)
|
| 20 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 21 |
+
args = parser.parse_args()
|
| 22 |
+
report = {"cache": str(args.cache.resolve()), "splits": {}}
|
| 23 |
+
for split in ("train", "val", "test"):
|
| 24 |
+
datasets = {
|
| 25 |
+
model: TachinEditorCache(args.cache, split, model,
|
| 26 |
+
train_or_val=split != "test", include_w3=model == "twla")
|
| 27 |
+
for model in ("smolvla", "vitra", "trex", "twla")
|
| 28 |
+
}
|
| 29 |
+
reference = datasets["smolvla"]
|
| 30 |
+
for model, dataset in datasets.items():
|
| 31 |
+
if not dataset.paired30 or not np.array_equal(dataset.rows, reference.rows):
|
| 32 |
+
raise AssertionError(f"{split}/{model} does not use the common paired rows")
|
| 33 |
+
counts = np.bincount(reference.rows[:, 1], minlength=3)
|
| 34 |
+
old_pressure = reference.common["pressure"]
|
| 35 |
+
old_future = reference.common["future_pressure"]
|
| 36 |
+
old_mask = reference.common["pressure_mask"]
|
| 37 |
+
old_future_mask = reference.common["future_pressure_mask"]
|
| 38 |
+
extra = reference.common["future_extra_pressure"]
|
| 39 |
+
extra_mask = reference.common["future_extra_pressure_mask"]
|
| 40 |
+
# Audit all rows. Exact equality is expected because paired windows are
|
| 41 |
+
# views of the original immutable values, not a second resampling pass.
|
| 42 |
+
for item, (query, tick) in enumerate(reference.rows):
|
| 43 |
+
sample = reference[item]
|
| 44 |
+
sequence = np.concatenate((old_pressure[query], old_future[query, :2]), axis=0)
|
| 45 |
+
sequence_mask = np.concatenate((old_mask[query], old_future_mask[query, :2]), axis=0)
|
| 46 |
+
targets = np.concatenate((old_future[query], extra[query]), axis=0)
|
| 47 |
+
target_masks = np.concatenate((old_future_mask[query], extra_mask[query]), axis=0)
|
| 48 |
+
if not np.array_equal(sample["pressure"].numpy(), sequence[tick:tick + 8]):
|
| 49 |
+
raise AssertionError("Tactile input window mismatch")
|
| 50 |
+
if not np.array_equal(sample["pressure_mask"].numpy(), sequence_mask[tick:tick + 8]):
|
| 51 |
+
raise AssertionError("Tactile input mask mismatch")
|
| 52 |
+
if not np.array_equal(sample["future_pressure"].numpy(), targets[tick + 2]):
|
| 53 |
+
raise AssertionError("Decision-relative W3 target mismatch")
|
| 54 |
+
if not np.array_equal(sample["future_pressure_mask"].numpy(), target_masks[tick + 2]):
|
| 55 |
+
raise AssertionError("Decision-relative W3 target mask mismatch")
|
| 56 |
+
batch = {name: value.unsqueeze(0) for name, value in sample.items()
|
| 57 |
+
if torch.is_tensor(value)}
|
| 58 |
+
mask = edit_mask(batch)[0]
|
| 59 |
+
editable = torch.where(mask.any(1))[0]
|
| 60 |
+
if len(editable) == 0 or int(editable[0]) != (0, 3, 6)[tick]:
|
| 61 |
+
raise AssertionError("Committed action prefix is editable")
|
| 62 |
+
report["splits"][split] = {
|
| 63 |
+
"plans": len(reference.common["target_action"]),
|
| 64 |
+
"paired_rows": len(reference),
|
| 65 |
+
"rows_by_decision_tick": counts.tolist(),
|
| 66 |
+
"models_share_exact_rows": True,
|
| 67 |
+
"causal_windows_exact": True,
|
| 68 |
+
"decision_relative_w3_targets_exact": True,
|
| 69 |
+
"first_editable_action": [0, 3, 6],
|
| 70 |
+
}
|
| 71 |
+
report["status"] = "PASS"
|
| 72 |
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
| 73 |
+
args.output.write_text(json.dumps(report, indent=2) + "\n")
|
| 74 |
+
print(json.dumps(report, indent=2))
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
if __name__ == "__main__":
|
| 78 |
+
main()
|
model/tachin_paired30_cache_20260923/code/scripts/build_tachin_paired30_cache.py
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Upgrade the native Tachin Table-1 cache to the paired-30 protocol.
|
| 2 |
+
|
| 3 |
+
The original cache already contains the eight frames available at plan time
|
| 4 |
+
and the next three 30-Hz tactile samples. Paired-30 needs the first two of
|
| 5 |
+
those samples as inputs at decision ticks 1 and 2, and needs two additional
|
| 6 |
+
samples (+4 and +5) as W3's decision-relative +100-ms targets. Base-policy,
|
| 7 |
+
memory, RGB-world, action-target, and split artifacts are reused by symlink.
|
| 8 |
+
"""
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import argparse
|
| 12 |
+
import hashlib
|
| 13 |
+
import json
|
| 14 |
+
import os
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
|
| 17 |
+
import numpy as np
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def load_npz(path: Path) -> dict[str, np.ndarray]:
|
| 21 |
+
with np.load(path, allow_pickle=False) as payload:
|
| 22 |
+
return {key: payload[key] for key in payload.files}
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def atomic_json(path: Path, value) -> None:
|
| 26 |
+
temporary = path.with_suffix(path.suffix + ".tmp")
|
| 27 |
+
temporary.write_text(json.dumps(value, indent=2) + "\n")
|
| 28 |
+
os.replace(temporary, path)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def sha256(path: Path) -> str:
|
| 32 |
+
digest = hashlib.sha256()
|
| 33 |
+
with path.open("rb") as stream:
|
| 34 |
+
for block in iter(lambda: stream.read(8 << 20), b""):
|
| 35 |
+
digest.update(block)
|
| 36 |
+
return digest.hexdigest()
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def link(source: Path, destination: Path) -> None:
|
| 40 |
+
if destination.is_symlink():
|
| 41 |
+
if destination.resolve() != source.resolve():
|
| 42 |
+
raise ValueError(f"Wrong existing symlink: {destination}")
|
| 43 |
+
return
|
| 44 |
+
if destination.exists():
|
| 45 |
+
raise FileExistsError(destination)
|
| 46 |
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
| 47 |
+
destination.symlink_to(source.resolve(), target_is_directory=source.is_dir())
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def main() -> None:
|
| 51 |
+
parser = argparse.ArgumentParser()
|
| 52 |
+
parser.add_argument("--source-cache", type=Path, required=True)
|
| 53 |
+
parser.add_argument("--adaptation-data", type=Path, required=True)
|
| 54 |
+
parser.add_argument("--benchmark", type=Path, required=True)
|
| 55 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 56 |
+
args = parser.parse_args()
|
| 57 |
+
|
| 58 |
+
root = Path(__file__).resolve().parents[1]
|
| 59 |
+
code = root / "artifacts/huggingface/vla-hand-baselines/model/tachin_adaptation_20260922/code"
|
| 60 |
+
import sys
|
| 61 |
+
sys.path.insert(0, str(code))
|
| 62 |
+
from data_preparation.tachin_benchmark.core import tactile_history
|
| 63 |
+
|
| 64 |
+
schema = json.loads((args.benchmark / "schema.json").read_text())
|
| 65 |
+
metadata = {
|
| 66 |
+
row["episode_id"]: row
|
| 67 |
+
for line in (args.benchmark / "episodes.jsonl").read_text().splitlines()
|
| 68 |
+
if (row := json.loads(line))
|
| 69 |
+
}
|
| 70 |
+
|
| 71 |
+
args.output.mkdir(parents=True, exist_ok=True)
|
| 72 |
+
link(args.source_cache / "statistics.npz", args.output / "statistics.npz")
|
| 73 |
+
for model in ("smolvla", "vitra", "trex", "twla"):
|
| 74 |
+
link(args.source_cache / "bases" / model, args.output / "bases" / model)
|
| 75 |
+
|
| 76 |
+
counts: dict[str, int] = {}
|
| 77 |
+
for split in ("train", "val", "test"):
|
| 78 |
+
source = args.source_cache / "shared" / split
|
| 79 |
+
destination = args.output / "shared" / split
|
| 80 |
+
destination.mkdir(parents=True, exist_ok=True)
|
| 81 |
+
rows_path = args.adaptation_data / split / "rows.json"
|
| 82 |
+
rows = json.loads(rows_path.read_text())
|
| 83 |
+
count = len(rows)
|
| 84 |
+
counts[split] = count
|
| 85 |
+
|
| 86 |
+
# Preserve the exact original sample identity and every immutable field.
|
| 87 |
+
for name in (
|
| 88 |
+
"target_action.npy", "action_mask.npy", "pressure.npy",
|
| 89 |
+
"pressure_mask.npy", "sample_valid.npy", "future_pressure.npy",
|
| 90 |
+
"future_pressure_mask.npy", "future_valid.npy", "beta.npy",
|
| 91 |
+
"identity.npz",
|
| 92 |
+
):
|
| 93 |
+
link(source / name, destination / name)
|
| 94 |
+
|
| 95 |
+
extra_path = destination / "future_extra_pressure.npy"
|
| 96 |
+
extra_mask_path = destination / "future_extra_pressure_mask.npy"
|
| 97 |
+
extra_valid_path = destination / "future_extra_valid.npy"
|
| 98 |
+
if (destination / "complete.json").is_file():
|
| 99 |
+
print(json.dumps({"split": split, "status": "already complete"}), flush=True)
|
| 100 |
+
continue
|
| 101 |
+
for path in (extra_path, extra_mask_path, extra_valid_path):
|
| 102 |
+
if path.exists():
|
| 103 |
+
raise FileExistsError(f"Incomplete paired-30 output exists: {path}")
|
| 104 |
+
pressure = np.lib.format.open_memmap(
|
| 105 |
+
extra_path, mode="w+", shape=(count, 2, 2, 880), dtype="float32")
|
| 106 |
+
pressure_mask = np.lib.format.open_memmap(
|
| 107 |
+
extra_mask_path, mode="w+", shape=(count, 2, 2, 880), dtype="bool")
|
| 108 |
+
valid = np.lib.format.open_memmap(
|
| 109 |
+
extra_valid_path, mode="w+", shape=(count, 2, 2), dtype="bool")
|
| 110 |
+
|
| 111 |
+
cached_recording = None
|
| 112 |
+
streams = None
|
| 113 |
+
for index, row in enumerate(rows):
|
| 114 |
+
meta = metadata[row["episode_id"]]
|
| 115 |
+
recording = meta["recording_id"]
|
| 116 |
+
if recording != cached_recording:
|
| 117 |
+
streams = [
|
| 118 |
+
load_npz(args.benchmark / "streams" / recording / f"{hand}.npz")
|
| 119 |
+
for hand in ("left", "right")
|
| 120 |
+
]
|
| 121 |
+
cached_recording = recording
|
| 122 |
+
timestamp = float(row["timestamp"])
|
| 123 |
+
for extra_index, frame_offset in enumerate((4, 5)):
|
| 124 |
+
future = tactile_history(
|
| 125 |
+
streams,
|
| 126 |
+
timestamp + frame_offset / 30.0,
|
| 127 |
+
frames=1,
|
| 128 |
+
history_hz=30.0,
|
| 129 |
+
max_age_s=float(schema["max_tactile_age_s"]),
|
| 130 |
+
)
|
| 131 |
+
pressure[index, extra_index] = future["pressure"][0]
|
| 132 |
+
pressure_mask[index, extra_index] = future["pressure_mask"][0]
|
| 133 |
+
valid[index, extra_index] = future["sample_mask"][0]
|
| 134 |
+
if index % 1000 == 0:
|
| 135 |
+
print(json.dumps({"split": split, "done": index, "total": count}), flush=True)
|
| 136 |
+
pressure.flush(); pressure_mask.flush(); valid.flush()
|
| 137 |
+
|
| 138 |
+
identity = dict(np.load(source / "identity.npz", allow_pickle=False))
|
| 139 |
+
if not np.array_equal(identity["sample_id"].astype(str),
|
| 140 |
+
np.asarray([row["sample_id"] for row in rows]).astype(str)):
|
| 141 |
+
raise ValueError(f"Sample identity mismatch in {split}")
|
| 142 |
+
atomic_json(destination / "complete.json", {
|
| 143 |
+
"schema": "tachin.table1.paired30.shared.v1",
|
| 144 |
+
"split": split,
|
| 145 |
+
"plans": count,
|
| 146 |
+
"rows_sha256": sha256(rows_path),
|
| 147 |
+
"decision_ticks": [0, 1, 2],
|
| 148 |
+
"decision_hz": 30.0,
|
| 149 |
+
"action_hz": 80.0,
|
| 150 |
+
"first_editable_action": [0, 3, 6],
|
| 151 |
+
"touch_input": "8-frame causal windows ending at plan + decision_tick/30 s",
|
| 152 |
+
"w3_tactile_target": "decision-relative +100 ms (+3 frames)",
|
| 153 |
+
"base_plan_reused_across_decisions": True,
|
| 154 |
+
"gated": False,
|
| 155 |
+
})
|
| 156 |
+
print(json.dumps({"split": split, "status": "complete", "plans": count}), flush=True)
|
| 157 |
+
|
| 158 |
+
atomic_json(args.output / "complete.json", {
|
| 159 |
+
"schema": "tachin.table1.editor_cache.paired30.v1",
|
| 160 |
+
"counts": counts,
|
| 161 |
+
"source_cache": str(args.source_cache.resolve()),
|
| 162 |
+
"benchmark": str(args.benchmark.resolve()),
|
| 163 |
+
"adaptation_data": str(args.adaptation_data.resolve()),
|
| 164 |
+
"vision_plan_hz": 10.0,
|
| 165 |
+
"tactile_input_hz": 30.0,
|
| 166 |
+
"editor_decision_hz": 30.0,
|
| 167 |
+
"decision_ticks": [0, 1, 2],
|
| 168 |
+
"action_hz": 80.0,
|
| 169 |
+
"gated": False,
|
| 170 |
+
})
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
if __name__ == "__main__":
|
| 174 |
+
main()
|
model/tachin_paired30_cache_20260923/code/scripts/cache_tachin_adapted_base.py
ADDED
|
@@ -0,0 +1,251 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Cache frozen Tachin-adapted SmolVLA, VITRA, or HumanTReX predictions."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import argparse
|
| 5 |
+
import hashlib
|
| 6 |
+
import importlib.util
|
| 7 |
+
import json
|
| 8 |
+
import os
|
| 9 |
+
import sys
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
|
| 12 |
+
import numpy as np
|
| 13 |
+
import torch
|
| 14 |
+
from torch.utils.data import DataLoader, Subset
|
| 15 |
+
|
| 16 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def install_py310_file_digest() -> None:
|
| 20 |
+
if hasattr(hashlib, "file_digest"):
|
| 21 |
+
return
|
| 22 |
+
def file_digest(stream, algorithm):
|
| 23 |
+
result = hashlib.new(algorithm)
|
| 24 |
+
for block in iter(lambda: stream.read(8 << 20), b""):
|
| 25 |
+
result.update(block)
|
| 26 |
+
return result
|
| 27 |
+
hashlib.file_digest = file_digest
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def sha256(path: Path) -> str:
|
| 31 |
+
with path.open("rb") as stream:
|
| 32 |
+
return hashlib.file_digest(stream, "sha256").hexdigest()
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def atomic_json(path: Path, payload) -> None:
|
| 36 |
+
temporary = path.with_suffix(path.suffix + ".tmp")
|
| 37 |
+
temporary.write_text(json.dumps(payload, indent=2, default=str) + "\n")
|
| 38 |
+
os.replace(temporary, path)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def per_row_noise(indices, device, seed):
|
| 42 |
+
return torch.stack([
|
| 43 |
+
torch.randn((16, 192), device=device,
|
| 44 |
+
generator=torch.Generator(device=device).manual_seed(seed + int(index)))
|
| 45 |
+
for index in indices
|
| 46 |
+
])
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def load_adapted_local(adaptation, checkpoint_path, model_name, data_root,
|
| 50 |
+
release, model_assets, device):
|
| 51 |
+
"""Restore against project-local frozen initialization/source assets."""
|
| 52 |
+
# transformers 5.17 moved this context manager out of modeling_utils, while
|
| 53 |
+
# the frozen Tachin release imports it from the former public location.
|
| 54 |
+
# Keep the archived release immutable and provide the compatibility alias
|
| 55 |
+
# in this runner instead.
|
| 56 |
+
if model_name != "trex":
|
| 57 |
+
import transformers.modeling_utils as modeling_utils
|
| 58 |
+
if not hasattr(modeling_utils, "no_init_weights"):
|
| 59 |
+
from transformers.initialization import no_init_weights
|
| 60 |
+
modeling_utils.no_init_weights = no_init_weights
|
| 61 |
+
checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=False)
|
| 62 |
+
initialization = {
|
| 63 |
+
"smolvla": model_assets / "smolvla_official/best.pt",
|
| 64 |
+
"vitra": model_assets / "vitra_original/best.pt",
|
| 65 |
+
"trex": release / "initialization/trex_g3/best.pt",
|
| 66 |
+
}[model_name]
|
| 67 |
+
if sha256(initialization) != checkpoint["initialization"]["initial_sha256"]:
|
| 68 |
+
raise ValueError(f"Project-local {model_name} initialization digest mismatch")
|
| 69 |
+
configuration = checkpoint["config"]
|
| 70 |
+
if model_name != "trex":
|
| 71 |
+
adaptation.ASSETS = model_assets
|
| 72 |
+
model, package, config, tokenizer, audit = adaptation.build(
|
| 73 |
+
model_name, initialization, checkpoint["statistics"], device,
|
| 74 |
+
not configuration["unfreeze_vlm"], data_root / "statistics.json")
|
| 75 |
+
else:
|
| 76 |
+
initial = torch.load(initialization, map_location="cpu", mmap=True, weights_only=False)
|
| 77 |
+
adaptation.check_initialization(initial)
|
| 78 |
+
spec = importlib.util.spec_from_file_location(
|
| 79 |
+
"tachin_human_trex_local", release / "code/training/trex_lora/model.py")
|
| 80 |
+
module = importlib.util.module_from_spec(spec)
|
| 81 |
+
spec.loader.exec_module(module)
|
| 82 |
+
config_values = initial["config"]
|
| 83 |
+
source = model_assets / "trex/source"
|
| 84 |
+
midtrain = model_assets / "trex/midtrain"
|
| 85 |
+
model = module.HumanTReX(str(source), str(midtrain), config_values["rank"],
|
| 86 |
+
config_values["alpha"], False)
|
| 87 |
+
model.load_adapter(initial["adapter"])
|
| 88 |
+
model.set_stats(checkpoint["statistics"])
|
| 89 |
+
from transformers import AutoProcessor
|
| 90 |
+
tokenizer = AutoProcessor.from_pretrained(str(midtrain / "processor"),
|
| 91 |
+
local_files_only=True)
|
| 92 |
+
tokenizer.tokenizer.padding_side = "left"
|
| 93 |
+
tokenizer.image_processor.size = {"shortest_edge": 4096, "longest_edge": 262144}
|
| 94 |
+
model.to(device)
|
| 95 |
+
package = config = None
|
| 96 |
+
audit = {"model": "trex", "initial_checkpoint": str(initialization.resolve()),
|
| 97 |
+
"source": str(source.resolve()), "midtrain": str(midtrain.resolve()),
|
| 98 |
+
"tactile_input": False, "editor": False, "world_model": False}
|
| 99 |
+
expected = {name for name, parameter in model.named_parameters() if parameter.requires_grad}
|
| 100 |
+
expected.update(name for name in model.state_dict()
|
| 101 |
+
if name in ("action_mean", "action_std", "state_mean", "state_std", "stats_loaded"))
|
| 102 |
+
if expected != set(checkpoint["trainable"]):
|
| 103 |
+
raise ValueError("Adapted trainable parameter set changed")
|
| 104 |
+
model.load_state_dict(checkpoint["trainable"], strict=False)
|
| 105 |
+
model.eval()
|
| 106 |
+
return model, package, config, tokenizer, audit
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
@torch.inference_mode()
|
| 110 |
+
def predict(model, package, batch, indices, device, seed):
|
| 111 |
+
noise = per_row_noise(indices, device, seed)
|
| 112 |
+
if package is None:
|
| 113 |
+
mask = batch["observed_mask"][:, None, :].expand(-1, 16, -1)
|
| 114 |
+
x = noise * mask
|
| 115 |
+
encoded = model.encode(batch)
|
| 116 |
+
for step in range(10):
|
| 117 |
+
time = torch.full((len(x),), 1 - step / 10, device=device)
|
| 118 |
+
x = (x - model.velocity(batch, x, time, encoded) / 10) * mask
|
| 119 |
+
return (x * model.action_std + model.action_mean) * mask
|
| 120 |
+
import train as adaptation
|
| 121 |
+
observations, _, _, _ = adaptation.arguments(batch, package)
|
| 122 |
+
normalized = model.predict_from_noise(observations, noise.to(batch["actions"].dtype))
|
| 123 |
+
return model.unnormalise_action(normalized)
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def main() -> None:
|
| 127 |
+
parser = argparse.ArgumentParser()
|
| 128 |
+
parser.add_argument("--model", choices=("smolvla", "vitra", "trex"), required=True)
|
| 129 |
+
parser.add_argument("--release", type=Path, required=True)
|
| 130 |
+
parser.add_argument("--data", type=Path, required=True)
|
| 131 |
+
parser.add_argument("--common", type=Path, required=True)
|
| 132 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 133 |
+
parser.add_argument("--model-assets", type=Path,
|
| 134 |
+
default=ROOT / "artifacts/model_assets")
|
| 135 |
+
parser.add_argument("--mano", type=Path)
|
| 136 |
+
parser.add_argument("--batch-size", type=int, default=8)
|
| 137 |
+
parser.add_argument("--workers", type=int, default=2)
|
| 138 |
+
parser.add_argument("--seed", type=int, default=20260922)
|
| 139 |
+
parser.add_argument("--limit", type=int, default=0)
|
| 140 |
+
parser.add_argument("--shard", type=int, default=-1)
|
| 141 |
+
parser.add_argument("--shards", type=int, default=1)
|
| 142 |
+
args = parser.parse_args()
|
| 143 |
+
if args.model == "vitra" and args.mano is None:
|
| 144 |
+
parser.error("--mano is required for VITRA native MANO FK")
|
| 145 |
+
if args.shard >= 0 and (args.limit or not 0 <= args.shard < args.shards):
|
| 146 |
+
parser.error("sharded cache requires --limit 0 and 0 <= shard < shards")
|
| 147 |
+
|
| 148 |
+
install_py310_file_digest()
|
| 149 |
+
code = args.release / "code"
|
| 150 |
+
trainer = code / "training/tachin_adaptation"
|
| 151 |
+
sys.path[:0] = [str(trainer), str(code), str(Path(__file__).resolve().parents[1] / "src")]
|
| 152 |
+
import train as adaptation
|
| 153 |
+
from tactile_editor.mano_action import ManoActionConverter
|
| 154 |
+
|
| 155 |
+
torch.set_num_threads(4)
|
| 156 |
+
torch.set_float32_matmul_precision("high")
|
| 157 |
+
device = torch.device("cuda:0")
|
| 158 |
+
torch.cuda.set_device(device)
|
| 159 |
+
checkpoint = args.release / args.model / "best.pt"
|
| 160 |
+
model, package, config, tokenizer, audit = load_adapted_local(
|
| 161 |
+
adaptation, checkpoint, args.model, args.data, args.release,
|
| 162 |
+
args.model_assets, device)
|
| 163 |
+
collator = adaptation.Collator(args.model, tokenizer, config)
|
| 164 |
+
converter = ManoActionConverter(args.mano, device=device) if args.model == "vitra" else None
|
| 165 |
+
|
| 166 |
+
for split in ("train", "val", "test"):
|
| 167 |
+
layout = "angle" if args.model == "vitra" else "keypoints"
|
| 168 |
+
dataset = adaptation.TachinDataset(args.data, split, layout)
|
| 169 |
+
common_ids = dict(np.load(args.common / "shared" / split / "identity.npz", allow_pickle=False))
|
| 170 |
+
source_ids = np.asarray([row["sample_id"] for row in dataset.rows])
|
| 171 |
+
if not np.array_equal(source_ids.astype(str), common_ids["sample_id"].astype(str)):
|
| 172 |
+
raise ValueError(f"{args.model}/{split} does not match common row identity")
|
| 173 |
+
count = len(dataset) if not args.limit else min(len(dataset), args.limit)
|
| 174 |
+
output = args.output / "bases" / args.model / split
|
| 175 |
+
output.mkdir(parents=True, exist_ok=True)
|
| 176 |
+
complete_path = output / ("smoke_complete.json" if args.limit else "complete.json")
|
| 177 |
+
if complete_path.exists():
|
| 178 |
+
print(json.dumps({"model": args.model, "split": split, "status": "already complete"}), flush=True)
|
| 179 |
+
continue
|
| 180 |
+
shard_complete = (output / f"complete_shard_{args.shard:02d}_of_{args.shards:02d}.json"
|
| 181 |
+
if args.shard >= 0 else None)
|
| 182 |
+
if shard_complete is not None and shard_complete.exists():
|
| 183 |
+
print(json.dumps({"model": args.model, "split": split, "shard": args.shard,
|
| 184 |
+
"status": "already complete"}), flush=True)
|
| 185 |
+
continue
|
| 186 |
+
prediction_path = output / ("smoke_base_action.npy" if args.limit else "base_action.npy")
|
| 187 |
+
marker_path = output / ("smoke_row_complete.npy" if args.limit else "row_complete.npy")
|
| 188 |
+
if prediction_path.exists():
|
| 189 |
+
predictions = np.load(prediction_path, mmap_mode="r+")
|
| 190 |
+
markers = np.load(marker_path, mmap_mode="r+")
|
| 191 |
+
if predictions.shape != (count, 16, 192) or markers.shape != (count,):
|
| 192 |
+
raise ValueError(f"Bad resumable cache schema: {output}")
|
| 193 |
+
else:
|
| 194 |
+
predictions = np.lib.format.open_memmap(prediction_path, mode="w+", dtype="float32", shape=(count, 16, 192))
|
| 195 |
+
markers = np.lib.format.open_memmap(marker_path, mode="w+", dtype="bool", shape=(count,))
|
| 196 |
+
markers[:] = False
|
| 197 |
+
markers.flush()
|
| 198 |
+
assigned = (np.arange(args.shard, count, args.shards, dtype=np.int64)
|
| 199 |
+
if args.shard >= 0 else np.arange(count, dtype=np.int64))
|
| 200 |
+
remaining = assigned[~markers[assigned]]
|
| 201 |
+
loader = DataLoader(Subset(dataset, remaining.tolist()), batch_size=args.batch_size,
|
| 202 |
+
shuffle=False, num_workers=args.workers, collate_fn=collator,
|
| 203 |
+
pin_memory=True, persistent_workers=args.workers > 0)
|
| 204 |
+
cursor = 0
|
| 205 |
+
for batch_index, batch in enumerate(loader):
|
| 206 |
+
size = len(batch["actions"])
|
| 207 |
+
positions = remaining[cursor:cursor + size]
|
| 208 |
+
batch = adaptation.to_device(batch, device)
|
| 209 |
+
with adaptation.precision(package):
|
| 210 |
+
values = predict(model, package, batch, positions, device, args.seed)
|
| 211 |
+
if converter is not None:
|
| 212 |
+
betas = np.asarray([dataset.rows[int(index)]["beta"] for index in positions], np.float32)
|
| 213 |
+
conventions = np.tile(np.asarray([["mano_left", "mano_right"]]), (size, 1))
|
| 214 |
+
values = converter(values.float(), betas=betas, conventions=conventions)
|
| 215 |
+
values = values.float().cpu().numpy()
|
| 216 |
+
if values.shape != (size, 16, 192) or not np.isfinite(values).all():
|
| 217 |
+
raise ValueError(f"Invalid {args.model} prediction batch")
|
| 218 |
+
predictions[positions] = values
|
| 219 |
+
predictions.flush()
|
| 220 |
+
markers[positions] = True
|
| 221 |
+
markers.flush()
|
| 222 |
+
cursor += size
|
| 223 |
+
if batch_index % 25 == 0:
|
| 224 |
+
print(json.dumps({"model": args.model, "split": split,
|
| 225 |
+
"done": int(markers.sum()), "total": count}), flush=True)
|
| 226 |
+
if not markers[assigned].all():
|
| 227 |
+
raise RuntimeError(f"Incomplete assigned cache rows: {output}")
|
| 228 |
+
payload = {
|
| 229 |
+
"schema": "tachin.table1.frozen_base.v1",
|
| 230 |
+
"model": args.model,
|
| 231 |
+
"split": split,
|
| 232 |
+
"rows": count,
|
| 233 |
+
"checkpoint": str(checkpoint.resolve()),
|
| 234 |
+
"checkpoint_sha256": sha256(checkpoint),
|
| 235 |
+
"adaptation_step": 1000,
|
| 236 |
+
"noise": f"per-row torch seed {args.seed} + zero-based common row index",
|
| 237 |
+
"output": "physical canonical [16,192] keypoint action",
|
| 238 |
+
"vitra_fk": "native MANO_LEFT/MANO_RIGHT" if converter is not None else None,
|
| 239 |
+
"audit": audit,
|
| 240 |
+
}
|
| 241 |
+
if shard_complete is not None:
|
| 242 |
+
atomic_json(shard_complete, {**payload, "assigned_rows": len(assigned),
|
| 243 |
+
"shard": args.shard, "shards": args.shards})
|
| 244 |
+
if markers.all():
|
| 245 |
+
atomic_json(complete_path, payload)
|
| 246 |
+
print(json.dumps({"model": args.model, "split": split, "shard": args.shard,
|
| 247 |
+
"status": "complete", "global_complete": bool(markers.all())}), flush=True)
|
| 248 |
+
|
| 249 |
+
|
| 250 |
+
if __name__ == "__main__":
|
| 251 |
+
main()
|
model/tachin_paired30_cache_20260923/code/scripts/cache_tachin_twla_w3.py
ADDED
|
@@ -0,0 +1,299 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Cache T-WLA-110k actions, recurrent memory, and W3 RGB-world inputs."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import argparse
|
| 5 |
+
import copy
|
| 6 |
+
import hashlib
|
| 7 |
+
import json
|
| 8 |
+
import os
|
| 9 |
+
import sys
|
| 10 |
+
import time
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
|
| 13 |
+
import numpy as np
|
| 14 |
+
import torch
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 18 |
+
V4 = ROOT / "ref/yuzhench_distill_editor/V4"
|
| 19 |
+
sys.path[:0] = [str(ROOT / "src"), str(ROOT / "ref/Tactile"), str(V4), str(V4 / "vla_hand_pose"), str(ROOT / "scripts")]
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def sha256(path: Path) -> str:
|
| 23 |
+
h = hashlib.sha256()
|
| 24 |
+
with path.open("rb") as stream:
|
| 25 |
+
for block in iter(lambda: stream.read(8 << 20), b""):
|
| 26 |
+
h.update(block)
|
| 27 |
+
return h.hexdigest()
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def array(root: Path, name: str, shape, dtype):
|
| 31 |
+
root.mkdir(parents=True, exist_ok=True)
|
| 32 |
+
path = root / f"{name}.npy"
|
| 33 |
+
if path.exists():
|
| 34 |
+
value = np.load(path, mmap_mode="r+")
|
| 35 |
+
if value.shape != tuple(shape) or value.dtype != np.dtype(dtype):
|
| 36 |
+
raise ValueError(f"Existing cache schema mismatch: {path}")
|
| 37 |
+
return value
|
| 38 |
+
return np.lib.format.open_memmap(path, mode="w+", shape=shape, dtype=dtype)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def atomic_json(path: Path, payload) -> None:
|
| 42 |
+
temporary = path.with_suffix(path.suffix + ".tmp")
|
| 43 |
+
temporary.write_text(json.dumps(payload, indent=2, default=str) + "\n")
|
| 44 |
+
os.replace(temporary, path)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def checkpoint_provenance(path: Path) -> dict:
|
| 48 |
+
"""Read cache-facing provenance without assuming the 110k initialization."""
|
| 49 |
+
payload = torch.load(path, map_location="cpu", mmap=True, weights_only=False)
|
| 50 |
+
step = payload.get("optimizer_step", payload.get("step"))
|
| 51 |
+
if step is None:
|
| 52 |
+
raise ValueError(f"Checkpoint does not declare a training/adaptation step: {path}")
|
| 53 |
+
adaptation = payload.get("adaptation")
|
| 54 |
+
result = {
|
| 55 |
+
"checkpoint_step": int(step),
|
| 56 |
+
"checkpoint_is_tachin_adapted": isinstance(adaptation, dict),
|
| 57 |
+
}
|
| 58 |
+
if isinstance(adaptation, dict):
|
| 59 |
+
result.update(
|
| 60 |
+
adaptation_protocol=adaptation.get("protocol"),
|
| 61 |
+
adaptation_initial_checkpoint=adaptation.get("checkpoint"),
|
| 62 |
+
adaptation_initial_sha256=adaptation.get("initial_sha256"),
|
| 63 |
+
adaptation_common_complete_sha256=adaptation.get("common_complete_sha256"),
|
| 64 |
+
)
|
| 65 |
+
del payload
|
| 66 |
+
return result
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
class PlannedDataset:
|
| 70 |
+
def __init__(self, base, common_indices, anchors):
|
| 71 |
+
self.base = base
|
| 72 |
+
self.common_indices = np.asarray(common_indices)
|
| 73 |
+
self.anchors = np.asarray(anchors)
|
| 74 |
+
|
| 75 |
+
def __len__(self):
|
| 76 |
+
return len(self.common_indices)
|
| 77 |
+
|
| 78 |
+
def __getitem__(self, index):
|
| 79 |
+
from tactile_editor.rgb_world_data import project_future_hand_roi
|
| 80 |
+
item = self.base[index]
|
| 81 |
+
unit = self.base.units[self.base._eval_plan[index][0]]
|
| 82 |
+
annotation, _, _ = self.base.cores[unit.source]._load_or_cache_episode(unit.episode)
|
| 83 |
+
anchor = int(self.anchors[index])
|
| 84 |
+
roi = np.zeros((2, 16, 16), bool)
|
| 85 |
+
roi_valid = np.zeros(2, bool)
|
| 86 |
+
hand = annotation.get(str(annotation["anno_type"]))
|
| 87 |
+
if isinstance(hand, dict):
|
| 88 |
+
for future_index, offset in enumerate((3, 6)):
|
| 89 |
+
if anchor + offset < len(annotation["video_decode_frame"]):
|
| 90 |
+
projected = project_future_hand_roi(
|
| 91 |
+
np.asarray(hand["joints_camspace"])[anchor + offset],
|
| 92 |
+
annotation["intrinsics"], source_hw=(1080, 1920),
|
| 93 |
+
)
|
| 94 |
+
roi[future_index] = projected.roi
|
| 95 |
+
roi_valid[future_index] = projected.valid
|
| 96 |
+
item.update(query_index=int(self.common_indices[index]), future_roi=roi,
|
| 97 |
+
future_roi_valid=roi_valid)
|
| 98 |
+
return item
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def main() -> None:
|
| 102 |
+
parser = argparse.ArgumentParser()
|
| 103 |
+
parser.add_argument("--checkpoint", type=Path, required=True)
|
| 104 |
+
parser.add_argument("--checkpoint-config", type=Path, required=True)
|
| 105 |
+
parser.add_argument("--world-checkpoint", type=Path, required=True)
|
| 106 |
+
parser.add_argument("--world-config", type=Path, required=True)
|
| 107 |
+
parser.add_argument("--annotations", type=Path, required=True)
|
| 108 |
+
parser.add_argument("--jpeg-root", type=Path, required=True)
|
| 109 |
+
parser.add_argument("--adaptation-data", type=Path, required=True)
|
| 110 |
+
parser.add_argument("--common", type=Path, required=True)
|
| 111 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 112 |
+
parser.add_argument("--shard", type=int, required=True)
|
| 113 |
+
parser.add_argument("--shards", type=int, default=4)
|
| 114 |
+
parser.add_argument("--batch-size", type=int, default=24)
|
| 115 |
+
parser.add_argument("--workers", type=int, default=4)
|
| 116 |
+
parser.add_argument("--seed", type=int, default=20260922)
|
| 117 |
+
parser.add_argument("--limit", type=int, default=0)
|
| 118 |
+
args = parser.parse_args()
|
| 119 |
+
if not 0 <= args.shard < args.shards:
|
| 120 |
+
raise ValueError("shard must be in [0, shards)")
|
| 121 |
+
|
| 122 |
+
checkpoint_metadata = checkpoint_provenance(args.checkpoint)
|
| 123 |
+
|
| 124 |
+
os.environ.update(METHOD_D_CHECKPOINT=str(args.checkpoint.resolve()),
|
| 125 |
+
METHOD_D_RUN_CONFIG=str(args.checkpoint_config.resolve()),
|
| 126 |
+
METHOD_D_SOURCE=str(V4 / "vla_hand_pose"))
|
| 127 |
+
from transformers import AutoTokenizer
|
| 128 |
+
from modal_build_full_editor_cache import load_backbone, apply_saved_config
|
| 129 |
+
from vla_hand_pose.v4_d_fm_0 import V4DConfig, MemoryBank, Observation, flow
|
| 130 |
+
from vla_hand_pose.dataloader import SequenceDataset, Source, make_collate, to_observations
|
| 131 |
+
from vla_hand_pose.world_model import dc_ae
|
| 132 |
+
from tactile_editor import EditorConfig, FusionConfig, TactileMethodD, Variant
|
| 133 |
+
|
| 134 |
+
torch.set_num_threads(3)
|
| 135 |
+
torch.set_float32_matmul_precision("high")
|
| 136 |
+
torch.manual_seed(args.seed)
|
| 137 |
+
device = torch.device("cuda:0")
|
| 138 |
+
backbone, config = load_backbone(device)
|
| 139 |
+
|
| 140 |
+
world_payload = torch.load(args.world_checkpoint, map_location="cpu", mmap=True, weights_only=False)
|
| 141 |
+
world_metadata = json.loads(args.world_config.read_text())
|
| 142 |
+
saved = world_payload.get("effective_config", world_metadata.get("effective_config", world_metadata.get("config")))
|
| 143 |
+
world_config = V4DConfig()
|
| 144 |
+
apply_saved_config(world_config, saved)
|
| 145 |
+
world_config.validate()
|
| 146 |
+
del world_payload
|
| 147 |
+
dataset_config = copy.deepcopy(config)
|
| 148 |
+
dataset_config.world = world_config.world
|
| 149 |
+
autoencoder = dc_ae.build(world_config.world.dcae, dtype=torch.float32).to(device)
|
| 150 |
+
fusion = FusionConfig(hidden_dim=config.expert.hidden_size, num_heads=config.expert.num_heads)
|
| 151 |
+
sampler = TactileMethodD(backbone, EditorConfig(variant=Variant.V0_BASE, fusion=fusion)).eval()
|
| 152 |
+
tokenizer = AutoTokenizer.from_pretrained(config.vlm.model_id)
|
| 153 |
+
source = Source(name="tachin", video_root="/unused",
|
| 154 |
+
annotation_file=str(args.annotations / "episode_frame_index.npz"),
|
| 155 |
+
label_folder=str(args.annotations / "episodic_annotations"),
|
| 156 |
+
jpeg_root=str(args.jpeg_root))
|
| 157 |
+
|
| 158 |
+
for split in ("train", "val", "test"):
|
| 159 |
+
rows = json.loads((args.adaptation_data / split / "rows.json").read_text())
|
| 160 |
+
identity = dict(np.load(args.common / "shared" / split / "identity.npz", allow_pickle=False))
|
| 161 |
+
if not np.array_equal(np.asarray([row["sample_id"] for row in rows]).astype(str),
|
| 162 |
+
identity["sample_id"].astype(str)):
|
| 163 |
+
raise ValueError(f"T-WLA {split} row identity mismatch")
|
| 164 |
+
all_indices = np.arange(args.shard, len(rows), args.shards, dtype=np.int64)
|
| 165 |
+
if args.limit:
|
| 166 |
+
all_indices = all_indices[:args.limit]
|
| 167 |
+
output = args.output / "bases/twla" / split / f"shard_{args.shard:02d}"
|
| 168 |
+
completion = output / ("smoke_complete.json" if args.limit else "complete.json")
|
| 169 |
+
if completion.exists():
|
| 170 |
+
print(json.dumps({"split": split, "shard": args.shard, "status": "already complete"}), flush=True)
|
| 171 |
+
continue
|
| 172 |
+
|
| 173 |
+
episodes = {str(rows[index]["episode_id"]) for index in all_indices}
|
| 174 |
+
dataset = SequenceDataset([source], cfg=dataset_config,
|
| 175 |
+
video_whitelist={"tachin": episodes}, tokenizer=tokenizer,
|
| 176 |
+
seed=0, load_images=True, training=False, deterministic=True)
|
| 177 |
+
units = {(unit.episode, unit.phase): index for index, unit in enumerate(dataset.units)}
|
| 178 |
+
plans = []
|
| 179 |
+
anchors = []
|
| 180 |
+
for common_index in all_indices:
|
| 181 |
+
row = rows[int(common_index)]
|
| 182 |
+
anchor = int(row["anchor_source_row"])
|
| 183 |
+
phase, relative = anchor % 3, anchor // 3
|
| 184 |
+
length = min(relative + 1, 9)
|
| 185 |
+
plans.append((units[(str(row["episode_id"]), phase)], relative - length + 1,
|
| 186 |
+
length, int(common_index)))
|
| 187 |
+
anchors.append(anchor)
|
| 188 |
+
dataset.set_eval_plan(plans)
|
| 189 |
+
wrapped = PlannedDataset(dataset, all_indices, anchors)
|
| 190 |
+
base_collate = make_collate(dataset_config, tokenizer)
|
| 191 |
+
def collate(items):
|
| 192 |
+
batch = base_collate(items)
|
| 193 |
+
for name in ("query_index", "future_roi", "future_roi_valid"):
|
| 194 |
+
batch[name] = torch.from_numpy(np.asarray([item[name] for item in items]))
|
| 195 |
+
return batch
|
| 196 |
+
|
| 197 |
+
count = len(all_indices)
|
| 198 |
+
fields = {
|
| 199 |
+
"base_action": ((count, 16, 192), "float32"),
|
| 200 |
+
"final_memory": ((count, 8, 960), "float16"),
|
| 201 |
+
"image_tokens": ((count, 64, 960), "float16"),
|
| 202 |
+
"clean_latent": ((count, 2, 32, 16, 16), "float16"),
|
| 203 |
+
"rgb_valid": ((count, 2), "bool"),
|
| 204 |
+
"future_roi": ((count, 2, 16, 16), "bool"),
|
| 205 |
+
"query_index": ((count,), "int64"),
|
| 206 |
+
}
|
| 207 |
+
values = {name: array(output, ("smoke_" if args.limit else "") + name, shape, dtype)
|
| 208 |
+
for name, (shape, dtype) in fields.items()}
|
| 209 |
+
marker = array(output, ("smoke_" if args.limit else "") + "row_complete", (count,), "bool")
|
| 210 |
+
remaining = np.flatnonzero(~marker)
|
| 211 |
+
loader = torch.utils.data.DataLoader(torch.utils.data.Subset(wrapped, remaining.tolist()),
|
| 212 |
+
batch_size=args.batch_size, shuffle=False, num_workers=args.workers,
|
| 213 |
+
collate_fn=collate, pin_memory=True, persistent_workers=args.workers > 0)
|
| 214 |
+
started = time.time()
|
| 215 |
+
cursor = 0
|
| 216 |
+
max_target_difference = 0.0
|
| 217 |
+
mask_mismatches = 0
|
| 218 |
+
target = np.load(args.common / "shared" / split / "target_action.npy", mmap_mode="r")
|
| 219 |
+
target_mask = np.load(args.common / "shared" / split / "action_mask.npy", mmap_mode="r")
|
| 220 |
+
with torch.inference_mode():
|
| 221 |
+
for batch_index, batch in enumerate(loader):
|
| 222 |
+
steps, step_valid, actions, action_mask = to_observations(batch, device, Observation, dtype=torch.bfloat16)
|
| 223 |
+
size = len(actions)
|
| 224 |
+
bank = MemoryBank(size, config.memory.bank_capacity, config.memory.num_slots,
|
| 225 |
+
config.memory.dim, device=device, dtype=backbone.vlm.dtype)
|
| 226 |
+
for time_index, observation in enumerate(steps[:-1]):
|
| 227 |
+
active = torch.nonzero(step_valid[:, time_index], as_tuple=True)[0]
|
| 228 |
+
if not active.numel():
|
| 229 |
+
continue
|
| 230 |
+
subset = observation if active.numel() == size else observation.index_select(active)
|
| 231 |
+
bank.append_rows(active, backbone.encode(subset, bank, rows=active))
|
| 232 |
+
final_memory, image_tokens = backbone.encode(steps[-1], bank, return_image_tokens=True)
|
| 233 |
+
generation_mask = flow.component_action_mask(size, 16, 192, config.flow, device)
|
| 234 |
+
query_indices = batch["query_index"].numpy()
|
| 235 |
+
noise = torch.stack([
|
| 236 |
+
torch.randn((16, 192), device=device, dtype=final_memory.dtype,
|
| 237 |
+
generator=torch.Generator(device=device).manual_seed(args.seed + int(index)))
|
| 238 |
+
for index in query_indices
|
| 239 |
+
])
|
| 240 |
+
normalized = sampler.sample_from_memory(
|
| 241 |
+
final_memory, generation_mask,
|
| 242 |
+
state_with_mask=(backbone.state_with_mask(steps[-1].state, steps[-1].state_mask)
|
| 243 |
+
if config.action.direct_state_to_expert else None),
|
| 244 |
+
noise=noise, num_steps=config.flow.num_inference_steps,
|
| 245 |
+
return_normalized=True,
|
| 246 |
+
).float()
|
| 247 |
+
physical = normalized * backbone.action_std.float() + backbone.action_mean.float()
|
| 248 |
+
future_images = batch["future_images"].to(device).permute(0, 1, 4, 2, 3).float().div_(127.5).sub_(1)
|
| 249 |
+
clean_latent = dc_ae.encode_targets(autoencoder, future_images, world_config.world.dcae)
|
| 250 |
+
positions = remaining[cursor:cursor + size]
|
| 251 |
+
difference = np.abs(actions.float().cpu().numpy() - target[query_indices])
|
| 252 |
+
common_mask = target_mask[query_indices]
|
| 253 |
+
max_target_difference = max(max_target_difference,
|
| 254 |
+
float(difference[common_mask].max(initial=0)))
|
| 255 |
+
mask_mismatches += int(np.count_nonzero(action_mask.cpu().numpy() != common_mask))
|
| 256 |
+
payload = {
|
| 257 |
+
"base_action": physical,
|
| 258 |
+
"final_memory": final_memory,
|
| 259 |
+
"image_tokens": image_tokens,
|
| 260 |
+
"clean_latent": clean_latent,
|
| 261 |
+
"rgb_valid": batch["future_valid"].bool() & batch["future_roi_valid"].bool(),
|
| 262 |
+
"future_roi": batch["future_roi"],
|
| 263 |
+
"query_index": batch["query_index"],
|
| 264 |
+
}
|
| 265 |
+
for name, value in payload.items():
|
| 266 |
+
if value.dtype == torch.bfloat16:
|
| 267 |
+
value = value.float()
|
| 268 |
+
result = value.detach().cpu().numpy()
|
| 269 |
+
if not np.isfinite(result).all():
|
| 270 |
+
raise ValueError(f"Nonfinite T-WLA cache field {name}")
|
| 271 |
+
values[name][positions] = result
|
| 272 |
+
values[name].flush()
|
| 273 |
+
marker[positions] = True
|
| 274 |
+
marker.flush()
|
| 275 |
+
cursor += size
|
| 276 |
+
if batch_index % 25 == 0:
|
| 277 |
+
print(json.dumps({"split": split, "shard": args.shard,
|
| 278 |
+
"done": int(marker.sum()), "total": count,
|
| 279 |
+
"rows_per_second": cursor / max(time.time() - started, 1e-6)}), flush=True)
|
| 280 |
+
if not marker.all():
|
| 281 |
+
raise RuntimeError(f"Incomplete T-WLA shard: {output}")
|
| 282 |
+
atomic_json(completion, {
|
| 283 |
+
"schema": "tachin.table1.twla_w3_cache.v1",
|
| 284 |
+
"split": split, "shard": args.shard, "shards": args.shards,
|
| 285 |
+
"rows": count, "checkpoint": str(args.checkpoint.resolve()),
|
| 286 |
+
"checkpoint_sha256": sha256(args.checkpoint), **checkpoint_metadata,
|
| 287 |
+
"memory_protocol": "up to eight available 10-Hz historical observations plus current",
|
| 288 |
+
"noise": f"per-row torch seed {args.seed} + zero-based common row index",
|
| 289 |
+
"world_checkpoint": str(args.world_checkpoint.resolve()),
|
| 290 |
+
"world_checkpoint_sha256": sha256(args.world_checkpoint),
|
| 291 |
+
"max_common_target_difference": max_target_difference,
|
| 292 |
+
"common_mask_mismatches": mask_mismatches,
|
| 293 |
+
"seconds": time.time() - started,
|
| 294 |
+
})
|
| 295 |
+
print(json.dumps({"split": split, "shard": args.shard, "status": "complete"}), flush=True)
|
| 296 |
+
|
| 297 |
+
|
| 298 |
+
if __name__ == "__main__":
|
| 299 |
+
main()
|
model/tachin_paired30_cache_20260923/code/scripts/summarize_tachin_paired30.py
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Validate and summarize the four completed Tachin paired-30 Table-1 runs."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import argparse
|
| 5 |
+
import json
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
import numpy as np
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def main() -> None:
|
| 12 |
+
parser = argparse.ArgumentParser()
|
| 13 |
+
parser.add_argument("--run-root", type=Path, required=True)
|
| 14 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 15 |
+
args = parser.parse_args()
|
| 16 |
+
metrics = ("translation_cm", "rotation_deg", "joint_mpjpe_cm")
|
| 17 |
+
payload = {"protocol": "Tachin paired-30", "models": {}}
|
| 18 |
+
for model in ("smolvla", "vitra", "trex", "twla"):
|
| 19 |
+
root = args.run_root / model
|
| 20 |
+
report = json.loads((root / "summary.json").read_text())
|
| 21 |
+
config = report["config"]
|
| 22 |
+
if config["editor_protocol"] != "paired-30":
|
| 23 |
+
raise AssertionError(f"{model}: wrong editor protocol")
|
| 24 |
+
if config["decision_ticks"] != [0, 1, 2] or config["editor_decision_hz"] != 30.0:
|
| 25 |
+
raise AssertionError(f"{model}: wrong decision cadence")
|
| 26 |
+
base = np.load(root / "test_base_action.npy") if (root / "test_base_action.npy").exists() else None
|
| 27 |
+
# Base is not separately written by the trainer; recover it from the
|
| 28 |
+
# zero-touch invariant check against the source policy below.
|
| 29 |
+
zero = np.load(root / "test_zero_touch_action.npy")
|
| 30 |
+
editor = np.load(root / "test_editor_action.npy")
|
| 31 |
+
shuffled = np.load(root / "test_shuffled_touch_action.npy")
|
| 32 |
+
source = Path(config["cache"]) / "bases" / model / "test"
|
| 33 |
+
if model == "twla":
|
| 34 |
+
base = np.empty_like(zero)
|
| 35 |
+
for shard in sorted(source.glob("shard_*")):
|
| 36 |
+
ids = np.load(shard / "query_index.npy")
|
| 37 |
+
base[ids] = np.load(shard / "base_action.npy")
|
| 38 |
+
else:
|
| 39 |
+
base = np.load(source / "base_action.npy")
|
| 40 |
+
zero_max = float(np.max(np.abs(zero - base)))
|
| 41 |
+
if zero_max > 2e-6:
|
| 42 |
+
raise AssertionError(f"{model}: reference input changes the base plan ({zero_max})")
|
| 43 |
+
test = report["test"]
|
| 44 |
+
payload["models"][model] = {
|
| 45 |
+
"training_budget_epochs": config["epochs"],
|
| 46 |
+
"selected_epoch": report["selected_epoch"],
|
| 47 |
+
"validation_action_loss": report["validation"]["action_loss"],
|
| 48 |
+
"base_metrics": test["metrics"]["base"],
|
| 49 |
+
"editor_metrics": test["metrics"]["editor"],
|
| 50 |
+
"shuffled_touch_metrics": test["metrics"]["shuffled_touch"],
|
| 51 |
+
"zero_touch_metrics": test["metrics"]["zero_touch"],
|
| 52 |
+
"table1_cells": test["table1_cells"],
|
| 53 |
+
"zero_touch_max_action_difference": zero_max,
|
| 54 |
+
"editor_vs_shuffled_action_mae": float(np.mean(np.abs(editor - shuffled))),
|
| 55 |
+
}
|
| 56 |
+
competitors = ("smolvla", "vitra", "trex")
|
| 57 |
+
comparison = {}
|
| 58 |
+
for metric in metrics:
|
| 59 |
+
best_model = min(competitors,
|
| 60 |
+
key=lambda name: payload["models"][name]["editor_metrics"][metric])
|
| 61 |
+
best = payload["models"][best_model]["editor_metrics"][metric]
|
| 62 |
+
twla = payload["models"]["twla"]["editor_metrics"][metric]
|
| 63 |
+
comparison[metric] = {
|
| 64 |
+
"best_competing_model": best_model,
|
| 65 |
+
"best_competing_error": best,
|
| 66 |
+
"twla_error": twla,
|
| 67 |
+
"twla_reduction_percent_vs_best_competitor": 100.0 * (best - twla) / best,
|
| 68 |
+
}
|
| 69 |
+
payload["twla_vs_best_competing_editor"] = comparison
|
| 70 |
+
payload["cache_audit"] = str((args.run_root.parent / "cache_audit.json").resolve())
|
| 71 |
+
payload["status"] = "PASS"
|
| 72 |
+
args.output.write_text(json.dumps(payload, indent=2) + "\n")
|
| 73 |
+
print(json.dumps(payload, indent=2))
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
if __name__ == "__main__":
|
| 77 |
+
main()
|
model/tachin_paired30_cache_20260923/code/scripts/train_tachin_table1_editor.py
ADDED
|
@@ -0,0 +1,539 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Train and evaluate one native Tachin tactile editor for paper Table 1."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import argparse
|
| 5 |
+
import fcntl
|
| 6 |
+
import json
|
| 7 |
+
import math
|
| 8 |
+
import os
|
| 9 |
+
import sys
|
| 10 |
+
import time
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
|
| 13 |
+
import numpy as np
|
| 14 |
+
import torch
|
| 15 |
+
from torch.utils.data import DataLoader, Dataset
|
| 16 |
+
|
| 17 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 18 |
+
V4 = ROOT / "ref/yuzhench_distill_editor/V4"
|
| 19 |
+
sys.path[:0] = [str(ROOT / "src"), str(V4), str(V4 / "vla_hand_pose"), str(ROOT / "scripts")]
|
| 20 |
+
|
| 21 |
+
from tactile_editor.action_layout import FULL_ACTION_GROUPS
|
| 22 |
+
from tactile_editor.action_objectives import per_sample_action_loss
|
| 23 |
+
from tactile_editor.native_tachin import (NativeResidualEditor, NativeTachinEncoder,
|
| 24 |
+
NativeTachinW3Editor)
|
| 25 |
+
from tactile_editor.rgb_world_editor import roi_weighted_world_loss
|
| 26 |
+
from tactile_editor.single_stage_world import masked_tactile_flow_loss
|
| 27 |
+
from tactile_editor.table1_experiment import summarize_physical
|
| 28 |
+
from tactile_editor.training import build_future_action_mask
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def atomic_json(path: Path, payload) -> None:
|
| 32 |
+
temporary = path.with_suffix(path.suffix + ".tmp")
|
| 33 |
+
temporary.write_text(json.dumps(payload, indent=2, default=str) + "\n")
|
| 34 |
+
os.replace(temporary, path)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
class TachinEditorCache(Dataset):
|
| 38 |
+
def __init__(self, root: Path, split: str, model: str, *, train_or_val: bool,
|
| 39 |
+
include_w3: bool):
|
| 40 |
+
self.root, self.split, self.model = root, split, model
|
| 41 |
+
shared = root / "shared" / split
|
| 42 |
+
if not (shared / "complete.json").is_file():
|
| 43 |
+
raise RuntimeError(f"Incomplete common cache: {shared}")
|
| 44 |
+
names = ("target_action", "action_mask", "pressure", "pressure_mask",
|
| 45 |
+
"sample_valid", "future_pressure", "future_pressure_mask", "future_valid")
|
| 46 |
+
self.common = {name: np.load(shared / f"{name}.npy", mmap_mode="r") for name in names}
|
| 47 |
+
self.paired30 = (shared / "future_extra_pressure.npy").is_file()
|
| 48 |
+
if self.paired30:
|
| 49 |
+
for name in ("future_extra_pressure", "future_extra_pressure_mask", "future_extra_valid"):
|
| 50 |
+
self.common[name] = np.load(shared / f"{name}.npy", mmap_mode="r")
|
| 51 |
+
self.identity = dict(np.load(shared / "identity.npz", allow_pickle=False))
|
| 52 |
+
count = len(self.common["target_action"])
|
| 53 |
+
self.locations = None
|
| 54 |
+
if model == "twla":
|
| 55 |
+
self.shards = []
|
| 56 |
+
locations = np.full((count, 2), -1, np.int64)
|
| 57 |
+
for shard_index, path in enumerate(sorted((root / "bases/twla" / split).glob("shard_*"))):
|
| 58 |
+
if not (path / "complete.json").is_file():
|
| 59 |
+
continue
|
| 60 |
+
fields = ["base_action", "final_memory", "query_index"]
|
| 61 |
+
if include_w3:
|
| 62 |
+
fields += ["image_tokens", "clean_latent", "rgb_valid", "future_roi"]
|
| 63 |
+
payload = {name: np.load(path / f"{name}.npy", mmap_mode="r") for name in fields}
|
| 64 |
+
ids = payload["query_index"]
|
| 65 |
+
if np.any(locations[ids, 0] >= 0):
|
| 66 |
+
raise ValueError("Duplicate T-WLA query indices")
|
| 67 |
+
locations[ids, 0] = len(self.shards)
|
| 68 |
+
locations[ids, 1] = np.arange(len(ids))
|
| 69 |
+
self.shards.append(payload)
|
| 70 |
+
if not self.shards or np.any(locations < 0):
|
| 71 |
+
raise RuntimeError(f"Incomplete T-WLA cache for {split}")
|
| 72 |
+
self.locations = locations
|
| 73 |
+
else:
|
| 74 |
+
path = root / "bases" / model / split
|
| 75 |
+
if not (path / "complete.json").is_file():
|
| 76 |
+
raise RuntimeError(f"Incomplete {model} cache for {split}")
|
| 77 |
+
self.base_action = np.load(path / "base_action.npy", mmap_mode="r")
|
| 78 |
+
if self.base_action.shape != (count, 16, 192):
|
| 79 |
+
raise ValueError("Frozen-base cache shape mismatch")
|
| 80 |
+
|
| 81 |
+
latest_hand = self.common["sample_valid"][:, -1] & self.common["pressure_mask"][:, -1].any(-1)
|
| 82 |
+
if self.paired30:
|
| 83 |
+
future_latest = (self.common["future_valid"][:, :2]
|
| 84 |
+
& self.common["future_pressure_mask"][:, :2].any(-1))
|
| 85 |
+
latest_hand = np.concatenate((latest_hand[:, None], future_latest), axis=1)
|
| 86 |
+
else:
|
| 87 |
+
latest_hand = latest_hand[:, None]
|
| 88 |
+
action_hand = np.stack((self.common["action_mask"][:, :, :69].any((1, 2)),
|
| 89 |
+
self.common["action_mask"][:, :, 69:138].any((1, 2))), 1)
|
| 90 |
+
self.latest_hand = np.asarray(latest_hand)
|
| 91 |
+
eligible = (latest_hand & action_hand[:, None]).any(2)
|
| 92 |
+
if self.paired30:
|
| 93 |
+
for tick, committed in enumerate((0, 3, 6)):
|
| 94 |
+
eligible[:, tick] &= self.common["action_mask"][:, committed:, :138].any((1, 2))
|
| 95 |
+
self.rows = np.argwhere(eligible)
|
| 96 |
+
else:
|
| 97 |
+
indices = np.flatnonzero(eligible[:, 0]) if train_or_val else np.arange(count)
|
| 98 |
+
self.rows = np.stack((indices, np.zeros_like(indices)), axis=1)
|
| 99 |
+
|
| 100 |
+
def __len__(self):
|
| 101 |
+
return len(self.rows)
|
| 102 |
+
|
| 103 |
+
def __getitem__(self, item):
|
| 104 |
+
index, tick = map(int, self.rows[item])
|
| 105 |
+
if self.model == "twla":
|
| 106 |
+
shard, row = self.locations[index]
|
| 107 |
+
base = self.shards[int(shard)]
|
| 108 |
+
output = {name: torch.from_numpy(np.array(value[int(row)], copy=True))
|
| 109 |
+
for name, value in base.items() if name != "query_index"}
|
| 110 |
+
else:
|
| 111 |
+
output = {"base_action": torch.from_numpy(np.array(self.base_action[index], copy=True))}
|
| 112 |
+
for name in ("target_action", "action_mask"):
|
| 113 |
+
output[name] = torch.from_numpy(np.array(self.common[name][index], copy=True))
|
| 114 |
+
if self.paired30:
|
| 115 |
+
pressure = np.concatenate((self.common["pressure"][index],
|
| 116 |
+
self.common["future_pressure"][index, :2]), axis=0)
|
| 117 |
+
pressure_mask = np.concatenate((self.common["pressure_mask"][index],
|
| 118 |
+
self.common["future_pressure_mask"][index, :2]), axis=0)
|
| 119 |
+
sample_valid = np.concatenate((self.common["sample_valid"][index],
|
| 120 |
+
self.common["future_valid"][index, :2]), axis=0)
|
| 121 |
+
future_pressure = np.concatenate((self.common["future_pressure"][index],
|
| 122 |
+
self.common["future_extra_pressure"][index]), axis=0)
|
| 123 |
+
future_pressure_mask = np.concatenate((self.common["future_pressure_mask"][index],
|
| 124 |
+
self.common["future_extra_pressure_mask"][index]), axis=0)
|
| 125 |
+
future_valid = np.concatenate((self.common["future_valid"][index],
|
| 126 |
+
self.common["future_extra_valid"][index]), axis=0)
|
| 127 |
+
output["pressure"] = torch.from_numpy(np.array(pressure[tick:tick + 8], copy=True))
|
| 128 |
+
output["pressure_mask"] = torch.from_numpy(np.array(pressure_mask[tick:tick + 8], copy=True))
|
| 129 |
+
output["sample_valid"] = torch.from_numpy(np.array(sample_valid[tick:tick + 8], copy=True))
|
| 130 |
+
output["future_pressure"] = torch.from_numpy(np.array(future_pressure[tick + 2], copy=True))
|
| 131 |
+
output["future_pressure_mask"] = torch.from_numpy(
|
| 132 |
+
np.array(future_pressure_mask[tick + 2], copy=True))
|
| 133 |
+
output["future_valid"] = torch.from_numpy(np.array(future_valid[tick + 2], copy=True))
|
| 134 |
+
else:
|
| 135 |
+
for name in ("pressure", "pressure_mask", "sample_valid"):
|
| 136 |
+
output[name] = torch.from_numpy(np.array(self.common[name][index], copy=True))
|
| 137 |
+
output["future_pressure"] = torch.from_numpy(
|
| 138 |
+
np.array(self.common["future_pressure"][index, 2], copy=True))
|
| 139 |
+
output["future_pressure_mask"] = torch.from_numpy(
|
| 140 |
+
np.array(self.common["future_pressure_mask"][index, 2], copy=True))
|
| 141 |
+
output["future_valid"] = torch.from_numpy(
|
| 142 |
+
np.array(self.common["future_valid"][index, 2], copy=True))
|
| 143 |
+
output["query_index"] = torch.tensor(index)
|
| 144 |
+
output["decision_tick"] = torch.tensor(tick)
|
| 145 |
+
output["decision_seconds"] = torch.tensor(tick / 30.0, dtype=torch.float32)
|
| 146 |
+
return output
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def edit_mask(batch):
|
| 150 |
+
mask = build_future_action_mask(batch["action_mask"].bool(), batch["decision_seconds"])
|
| 151 |
+
mask[..., 138:] = False
|
| 152 |
+
hand_valid = batch["sample_valid"][:, -1].bool() & batch["pressure_mask"][:, -1].any(-1)
|
| 153 |
+
mask[..., :69] &= hand_valid[:, 0, None, None]
|
| 154 |
+
mask[..., 69:138] &= hand_valid[:, 1, None, None]
|
| 155 |
+
return mask
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def move(batch, device):
|
| 159 |
+
return {name: value.to(device, non_blocking=True) for name, value in batch.items()}
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
def prepare_batch(raw, device, statistics):
|
| 163 |
+
batch = move(raw, device)
|
| 164 |
+
mean = statistics["action_mean"].to(device)
|
| 165 |
+
scale = statistics["action_std"].to(device)
|
| 166 |
+
batch["base_action"] = (batch["base_action"].float() - mean) / scale
|
| 167 |
+
batch["target_action"] = (batch["target_action"].float() - mean) / scale
|
| 168 |
+
return batch
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def make_model(model_name, statistics, device):
|
| 172 |
+
encoder = NativeTachinEncoder(statistics["pressure_mean"], statistics["pressure_scale"])
|
| 173 |
+
if model_name == "twla":
|
| 174 |
+
model = NativeTachinW3Editor(encoder, statistics["pressure_mean"],
|
| 175 |
+
statistics["pressure_scale"])
|
| 176 |
+
else:
|
| 177 |
+
model = NativeResidualEditor(encoder)
|
| 178 |
+
return model.to(device)
|
| 179 |
+
|
| 180 |
+
|
| 181 |
+
def forward(model, batch, *, auxiliary=False, rgb_head=None, generator=None):
|
| 182 |
+
mask = edit_mask(batch)
|
| 183 |
+
if isinstance(model, NativeResidualEditor):
|
| 184 |
+
result = model(batch["base_action"], mask, batch["pressure"].float(),
|
| 185 |
+
batch["pressure_mask"], batch["sample_valid"])
|
| 186 |
+
prediction, delta, allowed, tactile, hand_valid = result
|
| 187 |
+
return prediction, delta, allowed, tactile, hand_valid, None
|
| 188 |
+
kwargs = {}
|
| 189 |
+
if auxiliary:
|
| 190 |
+
future_pressure = batch["future_pressure"].float()
|
| 191 |
+
future_valid = batch["future_pressure_mask"].bool() & batch["future_valid"][:, :, None].bool()
|
| 192 |
+
latent = batch["clean_latent"].float()
|
| 193 |
+
kwargs = dict(
|
| 194 |
+
future_pressure=future_pressure,
|
| 195 |
+
future_cell_valid=future_valid,
|
| 196 |
+
tactile_tau=torch.rand(len(mask), device=mask.device, generator=generator),
|
| 197 |
+
tactile_noise=torch.randn((len(mask), 2, 880), device=mask.device, generator=generator),
|
| 198 |
+
rgb_world_head=rgb_head,
|
| 199 |
+
image_tokens=batch["image_tokens"].float(),
|
| 200 |
+
clean_latent=latent,
|
| 201 |
+
rgb_valid=batch["rgb_valid"].bool(),
|
| 202 |
+
rgb_tau=torch.rand(latent.shape[:2], device=mask.device, generator=generator),
|
| 203 |
+
rgb_noise=torch.randn(latent.shape, device=mask.device, generator=generator),
|
| 204 |
+
)
|
| 205 |
+
output = model(base_action=batch["base_action"], edit_mask=mask,
|
| 206 |
+
pressure=batch["pressure"].float(), cell_valid=batch["pressure_mask"],
|
| 207 |
+
sample_valid=batch["sample_valid"], final_memory=batch["final_memory"].float(),
|
| 208 |
+
compute_auxiliary=auxiliary, **kwargs)
|
| 209 |
+
return (output.final_action, output.delta_action, output.allowed_mask,
|
| 210 |
+
output.tactile_features, output.hand_valid, output)
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
def objective(model, batch, *, rgb_head, generator, lambda_rgb, lambda_tactile):
|
| 214 |
+
prediction, delta, allowed, _, _, output = forward(
|
| 215 |
+
model, batch, auxiliary=isinstance(model, NativeTachinW3Editor),
|
| 216 |
+
rgb_head=rgb_head, generator=generator)
|
| 217 |
+
active = allowed.any((1, 2))
|
| 218 |
+
action = per_sample_action_loss(prediction[active], batch["target_action"][active],
|
| 219 |
+
allowed[active], groups=FULL_ACTION_GROUPS).mean()
|
| 220 |
+
delta_l2 = delta.float().square().masked_fill(~allowed, 0).sum() / allowed.sum().clamp_min(1)
|
| 221 |
+
tactile = action.new_zeros(())
|
| 222 |
+
rgb = action.new_zeros(())
|
| 223 |
+
if output is not None:
|
| 224 |
+
tactile = masked_tactile_flow_loss(output.tactile_velocity, output.tactile_target,
|
| 225 |
+
output.tactile_valid)
|
| 226 |
+
if output.rgb.valid.any():
|
| 227 |
+
rgb = roi_weighted_world_loss(output.rgb, batch["future_roi"].bool(),
|
| 228 |
+
background_weight=0.1)
|
| 229 |
+
loss = action + 1e-4 * delta_l2 + lambda_rgb * rgb + lambda_tactile * tactile
|
| 230 |
+
return loss, {"action": float(action.detach()), "rgb": float(rgb.detach()),
|
| 231 |
+
"tactile": float(tactile.detach()), "delta_l2": float(delta_l2.detach())}
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
@torch.no_grad()
|
| 235 |
+
def validate(model, dataset, statistics, batch_size, workers, device):
|
| 236 |
+
model.eval()
|
| 237 |
+
total = base_total = 0.0
|
| 238 |
+
count = 0
|
| 239 |
+
for raw in DataLoader(dataset, batch_size=batch_size, num_workers=workers,
|
| 240 |
+
pin_memory=True, persistent_workers=workers > 0):
|
| 241 |
+
batch = prepare_batch(raw, device, statistics)
|
| 242 |
+
prediction, _, allowed, _, _, _ = forward(model, batch)
|
| 243 |
+
active = allowed.any((1, 2))
|
| 244 |
+
losses = per_sample_action_loss(prediction[active], batch["target_action"][active],
|
| 245 |
+
allowed[active], groups=FULL_ACTION_GROUPS)
|
| 246 |
+
bases = per_sample_action_loss(batch["base_action"][active], batch["target_action"][active],
|
| 247 |
+
allowed[active], groups=FULL_ACTION_GROUPS)
|
| 248 |
+
total += float(losses.sum())
|
| 249 |
+
base_total += float(bases.sum())
|
| 250 |
+
count += len(losses)
|
| 251 |
+
return {"action_loss": total / max(count, 1),
|
| 252 |
+
"base_action_loss": base_total / max(count, 1), "rows": count}
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
def base_actions(dataset):
|
| 256 |
+
count = len(dataset.common["target_action"])
|
| 257 |
+
if dataset.model != "twla":
|
| 258 |
+
return np.asarray(dataset.base_action)
|
| 259 |
+
result = np.empty((count, 16, 192), np.float32)
|
| 260 |
+
for shard in dataset.shards:
|
| 261 |
+
result[shard["query_index"]] = shard["base_action"]
|
| 262 |
+
return result
|
| 263 |
+
|
| 264 |
+
|
| 265 |
+
@torch.no_grad()
|
| 266 |
+
def infer_condition(model, dataset, statistics, batch_size, workers, device, condition,
|
| 267 |
+
permutation=None):
|
| 268 |
+
model.eval()
|
| 269 |
+
base = base_actions(dataset)
|
| 270 |
+
prediction = base.copy()
|
| 271 |
+
mean = statistics["action_mean"]
|
| 272 |
+
scale = statistics["action_std"]
|
| 273 |
+
loader = DataLoader(dataset, batch_size=batch_size, num_workers=workers,
|
| 274 |
+
pin_memory=True, persistent_workers=workers > 0)
|
| 275 |
+
row_offset = 0
|
| 276 |
+
for raw in loader:
|
| 277 |
+
indices = raw["query_index"].numpy()
|
| 278 |
+
if condition == "shuffled_touch":
|
| 279 |
+
donors = permutation[row_offset:row_offset + len(indices)]
|
| 280 |
+
for name in ("pressure", "pressure_mask", "sample_valid"):
|
| 281 |
+
raw[name] = torch.stack([dataset[int(donor)][name] for donor in donors])
|
| 282 |
+
elif condition == "zero_touch":
|
| 283 |
+
reference = statistics["pressure_mean"].view(1, 1, 2, 880)
|
| 284 |
+
raw["pressure"] = reference.expand_as(raw["pressure"]).clone()
|
| 285 |
+
batch = prepare_batch(raw, device, statistics)
|
| 286 |
+
values, _, allowed, _, _, _ = forward(model, batch)
|
| 287 |
+
physical = values.float().cpu() * scale + mean
|
| 288 |
+
physical = physical.numpy()
|
| 289 |
+
allowed = allowed.cpu().numpy()
|
| 290 |
+
for local, index in enumerate(indices):
|
| 291 |
+
prediction[index][allowed[local]] = physical[local][allowed[local]]
|
| 292 |
+
row_offset += len(indices)
|
| 293 |
+
return prediction
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
def final_test(model, dataset, statistics, args, device, output):
|
| 297 |
+
count = len(dataset.common["target_action"])
|
| 298 |
+
row_hands = dataset.latest_hand[dataset.rows[:, 0], dataset.rows[:, 1]]
|
| 299 |
+
codes = row_hands[:, 0].astype(np.int8) + 2 * row_hands[:, 1].astype(np.int8)
|
| 300 |
+
permutation = np.arange(len(dataset))
|
| 301 |
+
rng = np.random.default_rng(args.seed + 700)
|
| 302 |
+
for tick in np.unique(dataset.rows[:, 1]):
|
| 303 |
+
for code in (0, 1, 2, 3):
|
| 304 |
+
rows = np.flatnonzero((dataset.rows[:, 1] == tick) & (codes == code))
|
| 305 |
+
donors = rows.copy()
|
| 306 |
+
rng.shuffle(donors)
|
| 307 |
+
permutation[rows] = donors
|
| 308 |
+
base = base_actions(dataset)
|
| 309 |
+
predictions = {"base": base}
|
| 310 |
+
for condition in ("editor", "shuffled_touch", "zero_touch"):
|
| 311 |
+
predictions[condition] = infer_condition(
|
| 312 |
+
model, dataset, statistics, args.micro_batch, args.workers, device,
|
| 313 |
+
condition, permutation)
|
| 314 |
+
np.save(output / f"test_{condition}_action.npy", predictions[condition])
|
| 315 |
+
target = np.asarray(dataset.common["target_action"])
|
| 316 |
+
mask = np.asarray(dataset.common["action_mask"])
|
| 317 |
+
episodes = dataset.identity["episode_id"].astype(str)
|
| 318 |
+
environment = np.full(count, "Tachin")
|
| 319 |
+
metrics = {
|
| 320 |
+
name: summarize_physical(value, target, mask, episodes, environment,
|
| 321 |
+
horizon=8, dataset="tachin")["overall"]
|
| 322 |
+
for name, value in predictions.items()
|
| 323 |
+
}
|
| 324 |
+
metric_names = ("translation_cm", "rotation_deg", "joint_mpjpe_cm")
|
| 325 |
+
reduction = {name: metrics["base"][name] - metrics["editor"][name] for name in metric_names}
|
| 326 |
+
percentage = {name: 100 * reduction[name] / metrics["base"][name] for name in metric_names}
|
| 327 |
+
cells = {name: f"{reduction[name]:.2f} / ({percentage[name]:.2f}%)" for name in metric_names}
|
| 328 |
+
return {"horizon": "first 8 actions / 100 ms", "aggregation": "episode-equal",
|
| 329 |
+
"editor_protocol": "paired-30" if dataset.paired30 else "single-decision",
|
| 330 |
+
"decision_ticks": [0, 1, 2] if dataset.paired30 else [0],
|
| 331 |
+
"metrics": metrics, "physical_error_reduction": reduction,
|
| 332 |
+
"physical_error_reduction_percent": percentage, "table1_cells": cells}
|
| 333 |
+
|
| 334 |
+
|
| 335 |
+
def main() -> None:
|
| 336 |
+
parser = argparse.ArgumentParser()
|
| 337 |
+
parser.add_argument("--cache", type=Path, required=True)
|
| 338 |
+
parser.add_argument("--model", choices=("smolvla", "vitra", "trex", "twla"), required=True)
|
| 339 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 340 |
+
parser.add_argument("--world-checkpoint", type=Path)
|
| 341 |
+
parser.add_argument("--world-config", type=Path)
|
| 342 |
+
parser.add_argument("--lambda-rgb", type=float, default=0.1,
|
| 343 |
+
help="T-WLA W3 RGB-world objective weight")
|
| 344 |
+
parser.add_argument("--lambda-tactile", type=float, default=1e-4,
|
| 345 |
+
help="T-WLA W3 tactile-world objective weight")
|
| 346 |
+
parser.add_argument("--epochs", type=int, default=30,
|
| 347 |
+
help="Maximum epochs; validation early stopping can finish sooner")
|
| 348 |
+
parser.add_argument("--min-epochs", type=int, default=5)
|
| 349 |
+
parser.add_argument("--patience", type=int, default=5,
|
| 350 |
+
help="Stop after this many validation epochs without improvement")
|
| 351 |
+
parser.add_argument("--min-delta", type=float, default=1e-4,
|
| 352 |
+
help="Minimum absolute validation action-loss improvement")
|
| 353 |
+
parser.add_argument("--micro-batch", type=int, default=32)
|
| 354 |
+
parser.add_argument("--effective-batch", type=int, default=256)
|
| 355 |
+
parser.add_argument("--workers", type=int, default=4)
|
| 356 |
+
parser.add_argument("--seed", type=int, default=20260922)
|
| 357 |
+
parser.add_argument("--smoke-steps", type=int, default=0)
|
| 358 |
+
parser.add_argument("--resume", action="store_true")
|
| 359 |
+
args = parser.parse_args()
|
| 360 |
+
if not 1 <= args.min_epochs <= args.epochs:
|
| 361 |
+
raise ValueError("min-epochs must be in [1, epochs]")
|
| 362 |
+
if args.patience < 1 or args.min_delta < 0:
|
| 363 |
+
raise ValueError("patience must be >= 1 and min-delta must be non-negative")
|
| 364 |
+
if args.lambda_rgb < 0 or args.lambda_tactile < 0:
|
| 365 |
+
raise ValueError("world objective weights must be non-negative")
|
| 366 |
+
if args.effective_batch % args.micro_batch:
|
| 367 |
+
raise ValueError("effective batch must be divisible by micro batch")
|
| 368 |
+
if args.model == "twla" and (args.world_checkpoint is None or args.world_config is None):
|
| 369 |
+
parser.error("T-WLA W3 requires the RGB world checkpoint and config")
|
| 370 |
+
args.output.mkdir(parents=True, exist_ok=True)
|
| 371 |
+
lock = (args.output / ".train.lock").open("w")
|
| 372 |
+
fcntl.flock(lock, fcntl.LOCK_EX)
|
| 373 |
+
if (args.output / "summary.json").exists():
|
| 374 |
+
print(json.dumps({"model": args.model, "status": "already complete"}), flush=True)
|
| 375 |
+
return
|
| 376 |
+
torch.set_num_threads(3)
|
| 377 |
+
torch.set_float32_matmul_precision("high")
|
| 378 |
+
torch.manual_seed(args.seed)
|
| 379 |
+
np.random.seed(args.seed)
|
| 380 |
+
device = torch.device("cuda:0")
|
| 381 |
+
stats_np = dict(np.load(args.cache / "statistics.npz", allow_pickle=False))
|
| 382 |
+
statistics = {name: torch.from_numpy(value).float() for name, value in stats_np.items()
|
| 383 |
+
if name in ("pressure_mean", "pressure_scale", "action_mean", "action_std")}
|
| 384 |
+
train = TachinEditorCache(args.cache, "train", args.model, train_or_val=True,
|
| 385 |
+
include_w3=args.model == "twla")
|
| 386 |
+
validation = TachinEditorCache(args.cache, "val", args.model, train_or_val=True,
|
| 387 |
+
include_w3=args.model == "twla")
|
| 388 |
+
model = make_model(args.model, statistics, device)
|
| 389 |
+
rgb_head = None
|
| 390 |
+
if args.model == "twla":
|
| 391 |
+
from train_unified_w0_w3 import load_rgb_world_head
|
| 392 |
+
rgb_head, _ = load_rgb_world_head(args.world_checkpoint, args.world_config, device)
|
| 393 |
+
groups = model.optimizer_groups()
|
| 394 |
+
lambda_rgb, lambda_tactile = args.lambda_rgb, args.lambda_tactile
|
| 395 |
+
else:
|
| 396 |
+
if args.lambda_rgb != 0.1 or args.lambda_tactile != 1e-4:
|
| 397 |
+
parser.error("world objective weights apply only to --model twla")
|
| 398 |
+
groups = [{"params": model.action.parameters(), "lr": 1e-4, "name": "action"},
|
| 399 |
+
{"params": model.encoder.parameters(), "lr": 1e-5, "name": "encoder"},
|
| 400 |
+
{"params": model.coefficients.parameters(), "lr": 1e-4, "name": "coefficients"}]
|
| 401 |
+
lambda_rgb = lambda_tactile = 0.0
|
| 402 |
+
optimizer = torch.optim.AdamW(groups, weight_decay=0.01)
|
| 403 |
+
steps_per_epoch = math.ceil(len(train) / args.effective_batch)
|
| 404 |
+
total_steps = args.epochs * steps_per_epoch
|
| 405 |
+
warmup = min(100, max(total_steps - 1, 0))
|
| 406 |
+
scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lambda step:
|
| 407 |
+
(step + 1) / max(warmup, 1) if step < warmup else
|
| 408 |
+
0.1 + 0.45 * (1 + math.cos(math.pi * min((step - warmup) / max(total_steps - warmup, 1), 1))))
|
| 409 |
+
config = {**vars(args), "train_rows": len(train), "validation_rows": len(validation),
|
| 410 |
+
"split": "original Tachin episode split", "gated": False,
|
| 411 |
+
"normalization": "shared original-train-only", "horizon": "8 actions / 100 ms",
|
| 412 |
+
"editor_protocol": "paired-30" if train.paired30 else "single-decision",
|
| 413 |
+
"vision_plan_hz": 10.0,
|
| 414 |
+
"tactile_input_hz": 30.0 if train.paired30 else 10.0,
|
| 415 |
+
"editor_decision_hz": 30.0 if train.paired30 else 10.0,
|
| 416 |
+
"decision_ticks": [0, 1, 2] if train.paired30 else [0],
|
| 417 |
+
"editor": "native 15-region 880-cell", "twla_variant": "W3" if args.model == "twla" else None,
|
| 418 |
+
"lambda_rgb": lambda_rgb, "lambda_tactile": lambda_tactile}
|
| 419 |
+
atomic_json(args.output / "config.json", config)
|
| 420 |
+
best = float("inf")
|
| 421 |
+
step = 0
|
| 422 |
+
first_epoch = 0
|
| 423 |
+
epochs_without_improvement = 0
|
| 424 |
+
started = time.time()
|
| 425 |
+
accumulation = args.effective_batch // args.micro_batch
|
| 426 |
+
generator = torch.Generator(device=device).manual_seed(args.seed + 1)
|
| 427 |
+
if args.resume:
|
| 428 |
+
resume_path = args.output / "last.pt"
|
| 429 |
+
if not resume_path.is_file():
|
| 430 |
+
raise FileNotFoundError(f"--resume requested but {resume_path} is missing")
|
| 431 |
+
checkpoint = torch.load(resume_path, map_location=device, weights_only=False)
|
| 432 |
+
required = {"model", "optimizer", "scheduler", "epoch", "step", "best",
|
| 433 |
+
"epochs_without_improvement"}
|
| 434 |
+
missing = required - set(checkpoint)
|
| 435 |
+
if missing:
|
| 436 |
+
raise ValueError(
|
| 437 |
+
f"{resume_path} predates resumable checkpoints and is missing {sorted(missing)}; "
|
| 438 |
+
"start a new output directory for a controlled early-stopping run")
|
| 439 |
+
model.load_state_dict(checkpoint["model"])
|
| 440 |
+
optimizer.load_state_dict(checkpoint["optimizer"])
|
| 441 |
+
scheduler.load_state_dict(checkpoint["scheduler"])
|
| 442 |
+
first_epoch = int(checkpoint["epoch"])
|
| 443 |
+
step = int(checkpoint["step"])
|
| 444 |
+
best = float(checkpoint["best"])
|
| 445 |
+
epochs_without_improvement = int(checkpoint["epochs_without_improvement"])
|
| 446 |
+
if checkpoint.get("aux_generator_state") is not None:
|
| 447 |
+
# torch.load(map_location=device) also moves the RNG byte state,
|
| 448 |
+
# while Generator.set_state requires a CPU ByteTensor.
|
| 449 |
+
generator.set_state(checkpoint["aux_generator_state"].cpu())
|
| 450 |
+
print(json.dumps({"model": args.model, "status": "resumed",
|
| 451 |
+
"next_epoch": first_epoch + 1, "step": step,
|
| 452 |
+
"best_validation_action_loss": best}), flush=True)
|
| 453 |
+
stopped_early = False
|
| 454 |
+
for epoch in range(first_epoch, args.epochs):
|
| 455 |
+
model.train()
|
| 456 |
+
loader = DataLoader(train, batch_size=args.micro_batch, shuffle=True,
|
| 457 |
+
generator=torch.Generator().manual_seed(args.seed + epoch),
|
| 458 |
+
num_workers=args.workers, pin_memory=True,
|
| 459 |
+
persistent_workers=args.workers > 0)
|
| 460 |
+
optimizer.zero_grad(set_to_none=True)
|
| 461 |
+
window = 0
|
| 462 |
+
sums = np.zeros(4)
|
| 463 |
+
seen = 0
|
| 464 |
+
for batch_index, raw in enumerate(loader):
|
| 465 |
+
batch = prepare_batch(raw, device, statistics)
|
| 466 |
+
with torch.autocast("cuda", dtype=torch.bfloat16):
|
| 467 |
+
loss, parts = objective(model, batch, rgb_head=rgb_head, generator=generator,
|
| 468 |
+
lambda_rgb=lambda_rgb, lambda_tactile=lambda_tactile)
|
| 469 |
+
count = len(batch["base_action"])
|
| 470 |
+
(loss * count / args.effective_batch).backward()
|
| 471 |
+
window += count
|
| 472 |
+
seen += count
|
| 473 |
+
sums += np.asarray(list(parts.values())) * count
|
| 474 |
+
if (batch_index + 1) % accumulation == 0 or batch_index + 1 == len(loader):
|
| 475 |
+
if window != args.effective_batch:
|
| 476 |
+
for parameter in model.parameters():
|
| 477 |
+
if parameter.grad is not None:
|
| 478 |
+
parameter.grad.mul_(args.effective_batch / window)
|
| 479 |
+
gradient = torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0,
|
| 480 |
+
error_if_nonfinite=True)
|
| 481 |
+
optimizer.step(); scheduler.step(); optimizer.zero_grad(set_to_none=True)
|
| 482 |
+
window = 0
|
| 483 |
+
step += 1
|
| 484 |
+
if step % 25 == 0 or args.smoke_steps:
|
| 485 |
+
print(json.dumps({"model": args.model, "step": step,
|
| 486 |
+
"total_steps": total_steps, "loss": parts,
|
| 487 |
+
"gradient": float(gradient)}), flush=True)
|
| 488 |
+
if args.smoke_steps and step >= args.smoke_steps:
|
| 489 |
+
atomic_json(args.output / "smoke.json", {"steps": step, "loss": parts,
|
| 490 |
+
"gradient": float(gradient)})
|
| 491 |
+
return
|
| 492 |
+
validation_metrics = validate(model, validation, statistics, args.micro_batch,
|
| 493 |
+
args.workers, device)
|
| 494 |
+
improved = validation_metrics["action_loss"] < best - args.min_delta
|
| 495 |
+
if improved:
|
| 496 |
+
best = validation_metrics["action_loss"]
|
| 497 |
+
epochs_without_improvement = 0
|
| 498 |
+
else:
|
| 499 |
+
epochs_without_improvement += 1
|
| 500 |
+
checkpoint = {"model": model.state_dict(), "optimizer": optimizer.state_dict(),
|
| 501 |
+
"scheduler": scheduler.state_dict(), "epoch": epoch + 1, "step": step,
|
| 502 |
+
"validation": validation_metrics, "config": config,
|
| 503 |
+
"statistics": statistics, "best": best,
|
| 504 |
+
"epochs_without_improvement": epochs_without_improvement,
|
| 505 |
+
"aux_generator_state": generator.get_state()}
|
| 506 |
+
torch.save(checkpoint, args.output / "last.pt")
|
| 507 |
+
if improved:
|
| 508 |
+
torch.save(checkpoint, args.output / "best.pt")
|
| 509 |
+
record = {"epoch": epoch + 1, "step": step,
|
| 510 |
+
"train": dict(zip(("action", "rgb", "tactile", "delta_l2"),
|
| 511 |
+
(sums / max(seen, 1)).tolist())),
|
| 512 |
+
"validation": validation_metrics, "improved": improved,
|
| 513 |
+
"epochs_without_improvement": epochs_without_improvement,
|
| 514 |
+
"seconds": time.time() - started}
|
| 515 |
+
with (args.output / "epochs.jsonl").open("a") as stream:
|
| 516 |
+
stream.write(json.dumps(record) + "\n")
|
| 517 |
+
print(json.dumps({"model": args.model, **record}), flush=True)
|
| 518 |
+
if epoch + 1 >= args.min_epochs and epochs_without_improvement >= args.patience:
|
| 519 |
+
stopped_early = True
|
| 520 |
+
print(json.dumps({"model": args.model, "status": "early_stop",
|
| 521 |
+
"epoch": epoch + 1, "patience": args.patience,
|
| 522 |
+
"best_validation_action_loss": best}), flush=True)
|
| 523 |
+
break
|
| 524 |
+
selected = torch.load(args.output / "best.pt", map_location=device, weights_only=False)
|
| 525 |
+
model.load_state_dict(selected["model"])
|
| 526 |
+
test = TachinEditorCache(args.cache, "test", args.model, train_or_val=False,
|
| 527 |
+
include_w3=args.model == "twla")
|
| 528 |
+
result = final_test(model, test, statistics, args, device, args.output)
|
| 529 |
+
atomic_json(args.output / "summary.json", {
|
| 530 |
+
"model": args.model, "selected_epoch": selected["epoch"],
|
| 531 |
+
"selected_step": selected["step"], "validation": selected["validation"],
|
| 532 |
+
"test": result, "seconds": time.time() - started, "config": config,
|
| 533 |
+
"stopped_early": stopped_early})
|
| 534 |
+
print(json.dumps({"model": args.model, "status": "complete",
|
| 535 |
+
"table1": result["table1_cells"]}), flush=True)
|
| 536 |
+
|
| 537 |
+
|
| 538 |
+
if __name__ == "__main__":
|
| 539 |
+
main()
|
model/tachin_paired30_cache_20260923/code/src/tactile_editor/action_layout.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Canonical Method-D action layout shared by training and evaluation.
|
| 2 |
+
|
| 3 |
+
The VLA emits 192 values per action step. The first 138 values are the two
|
| 4 |
+
69-dimensional hand actions; the final 54 values are padding and are never
|
| 5 |
+
editable or scored. Keeping this contract in one module prevents model rows
|
| 6 |
+
from silently using different masks for Table 1.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
from dataclasses import dataclass
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
ACTION_DIM = 192
|
| 17 |
+
CHUNK_SIZE = 16
|
| 18 |
+
VALID_ACTION_DIM = 138
|
| 19 |
+
HAND_ACTION_DIM = 69
|
| 20 |
+
|
| 21 |
+
LEFT_TRANSLATION = slice(0, 3)
|
| 22 |
+
LEFT_ROTATION = slice(3, 6)
|
| 23 |
+
LEFT_ARTICULATION = slice(6, 69)
|
| 24 |
+
RIGHT_TRANSLATION = slice(69, 72)
|
| 25 |
+
RIGHT_ROTATION = slice(72, 75)
|
| 26 |
+
RIGHT_ARTICULATION = slice(75, 138)
|
| 27 |
+
|
| 28 |
+
TRANSLATION_GROUPS = ((0, 3), (69, 72))
|
| 29 |
+
ROTATION_GROUPS = ((3, 6), (72, 75))
|
| 30 |
+
ARTICULATION_GROUPS = ((6, 69), (75, 138))
|
| 31 |
+
FULL_ACTION_GROUPS = (
|
| 32 |
+
(0, 3),
|
| 33 |
+
(3, 6),
|
| 34 |
+
(6, 69),
|
| 35 |
+
(69, 72),
|
| 36 |
+
(72, 75),
|
| 37 |
+
(75, 138),
|
| 38 |
+
)
|
| 39 |
+
FULL_HAND_GROUPS = ((0, 69), (69, 138))
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
@dataclass(frozen=True)
|
| 43 |
+
class ActionLayout:
|
| 44 |
+
"""Shape and physical component metadata for a normalized action chunk."""
|
| 45 |
+
|
| 46 |
+
chunk_size: int = CHUNK_SIZE
|
| 47 |
+
action_dim: int = ACTION_DIM
|
| 48 |
+
valid_action_dim: int = VALID_ACTION_DIM
|
| 49 |
+
|
| 50 |
+
def validate(self, action: torch.Tensor, *, name: str = "action") -> None:
|
| 51 |
+
if action.ndim < 2 or tuple(action.shape[-2:]) != (
|
| 52 |
+
self.chunk_size,
|
| 53 |
+
self.action_dim,
|
| 54 |
+
):
|
| 55 |
+
raise ValueError(
|
| 56 |
+
f"{name} must end in [{self.chunk_size},{self.action_dim}], "
|
| 57 |
+
f"got {tuple(action.shape)}"
|
| 58 |
+
)
|
| 59 |
+
|
| 60 |
+
def physical_mask(
|
| 61 |
+
self,
|
| 62 |
+
shape: tuple[int, ...],
|
| 63 |
+
*,
|
| 64 |
+
device: torch.device | None = None,
|
| 65 |
+
groups: tuple[tuple[int, int], ...] = FULL_ACTION_GROUPS,
|
| 66 |
+
) -> torch.Tensor:
|
| 67 |
+
if len(shape) < 2 or shape[-2:] != (self.chunk_size, self.action_dim):
|
| 68 |
+
raise ValueError("mask shape must end in [16,192]")
|
| 69 |
+
result = torch.zeros(shape, dtype=torch.bool, device=device)
|
| 70 |
+
for start, stop in groups:
|
| 71 |
+
if not 0 <= start < stop <= self.valid_action_dim:
|
| 72 |
+
raise ValueError("action group is outside the valid action range")
|
| 73 |
+
result[..., start:stop] = True
|
| 74 |
+
return result
|
| 75 |
+
|
| 76 |
+
def apply_edit_scope(
|
| 77 |
+
self,
|
| 78 |
+
action_mask: torch.Tensor,
|
| 79 |
+
*,
|
| 80 |
+
groups: tuple[tuple[int, int], ...] = FULL_ACTION_GROUPS,
|
| 81 |
+
) -> torch.Tensor:
|
| 82 |
+
"""Intersect a runtime mask with physical components and padding mask."""
|
| 83 |
+
|
| 84 |
+
self.validate(action_mask, name="action_mask")
|
| 85 |
+
return action_mask.bool() & self.physical_mask(
|
| 86 |
+
tuple(action_mask.shape), device=action_mask.device, groups=groups
|
| 87 |
+
)
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
DEFAULT_ACTION_LAYOUT = ActionLayout()
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
@dataclass(frozen=True)
|
| 94 |
+
class ActionNormalizer:
|
| 95 |
+
"""Frozen affine statistics used by every model row."""
|
| 96 |
+
|
| 97 |
+
mean: torch.Tensor
|
| 98 |
+
std: torch.Tensor
|
| 99 |
+
identity: str = "unlabelled"
|
| 100 |
+
|
| 101 |
+
def __post_init__(self) -> None:
|
| 102 |
+
if tuple(self.mean.shape) != (ACTION_DIM,) or tuple(self.std.shape) != (
|
| 103 |
+
ACTION_DIM,
|
| 104 |
+
):
|
| 105 |
+
raise ValueError("action statistics must have shape [192]")
|
| 106 |
+
if not torch.isfinite(self.mean).all() or not torch.isfinite(self.std).all():
|
| 107 |
+
raise ValueError("action statistics must be finite")
|
| 108 |
+
if (self.std <= 0).any():
|
| 109 |
+
raise ValueError("action standard deviations must be positive")
|
| 110 |
+
if not self.identity:
|
| 111 |
+
raise ValueError("normalizer identity must be non-empty")
|
| 112 |
+
|
| 113 |
+
def normalize(self, action: torch.Tensor) -> torch.Tensor:
|
| 114 |
+
DEFAULT_ACTION_LAYOUT.validate(action)
|
| 115 |
+
mean = self.mean.to(device=action.device, dtype=action.dtype)
|
| 116 |
+
std = self.std.to(device=action.device, dtype=action.dtype)
|
| 117 |
+
return (action - mean) / std
|
| 118 |
+
|
| 119 |
+
def denormalize(self, action: torch.Tensor) -> torch.Tensor:
|
| 120 |
+
DEFAULT_ACTION_LAYOUT.validate(action)
|
| 121 |
+
mean = self.mean.to(device=action.device, dtype=action.dtype)
|
| 122 |
+
std = self.std.to(device=action.device, dtype=action.dtype)
|
| 123 |
+
return action * std + mean
|
model/tachin_paired30_cache_20260923/code/src/tactile_editor/action_objectives.py
ADDED
|
@@ -0,0 +1,233 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Group-balanced action objectives for tactile-editor experiments.
|
| 2 |
+
|
| 3 |
+
All public loss functions return one value per input sample. Action objectives
|
| 4 |
+
require every sample to contain at least one supervised value. Motion
|
| 5 |
+
objectives also return a validity mask because an editable suffix can contain no
|
| 6 |
+
adjacent valid pair.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import math
|
| 12 |
+
from collections.abc import Sequence
|
| 13 |
+
|
| 14 |
+
import torch
|
| 15 |
+
from torch.nn import functional as F
|
| 16 |
+
|
| 17 |
+
from .training import ACTION_GROUPS
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
ActionGroups = Sequence[tuple[int, int]]
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def _validate_action_tensors(
|
| 24 |
+
prediction: torch.Tensor,
|
| 25 |
+
target: torch.Tensor,
|
| 26 |
+
mask: torch.Tensor,
|
| 27 |
+
) -> torch.Tensor:
|
| 28 |
+
if prediction.ndim != 3:
|
| 29 |
+
raise ValueError("prediction must be [B,T,A]")
|
| 30 |
+
if target.shape != prediction.shape or mask.shape != prediction.shape:
|
| 31 |
+
raise ValueError("prediction, target, and mask must have matching [B,T,A] shapes")
|
| 32 |
+
if not prediction.is_floating_point() or not target.is_floating_point():
|
| 33 |
+
raise TypeError("prediction and target must be floating-point tensors")
|
| 34 |
+
if prediction.device != target.device or prediction.device != mask.device:
|
| 35 |
+
raise ValueError("prediction, target, and mask must be on the same device")
|
| 36 |
+
return mask.bool()
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _validate_groups(groups: ActionGroups, action_dim: int) -> tuple[tuple[int, int], ...]:
|
| 40 |
+
groups = tuple(groups)
|
| 41 |
+
if not groups:
|
| 42 |
+
raise ValueError("at least one action group is required")
|
| 43 |
+
occupied: set[int] = set()
|
| 44 |
+
for start, stop in groups:
|
| 45 |
+
if not 0 <= start < stop <= action_dim:
|
| 46 |
+
raise ValueError("action group outside the action dimension")
|
| 47 |
+
indices = set(range(start, stop))
|
| 48 |
+
if occupied & indices:
|
| 49 |
+
raise ValueError("action groups must not overlap")
|
| 50 |
+
occupied |= indices
|
| 51 |
+
return groups
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _validate_timestep_weights(
|
| 55 |
+
timestep_weights: torch.Tensor,
|
| 56 |
+
prediction: torch.Tensor,
|
| 57 |
+
) -> torch.Tensor:
|
| 58 |
+
expected = (*prediction.shape[:2], 1)
|
| 59 |
+
if tuple(timestep_weights.shape) != expected:
|
| 60 |
+
raise ValueError(f"timestep_weights must have shape {expected}")
|
| 61 |
+
weights = timestep_weights.to(device=prediction.device, dtype=prediction.dtype)
|
| 62 |
+
if not torch.isfinite(weights).all() or (weights < 0).any():
|
| 63 |
+
raise ValueError("timestep_weights must be finite and non-negative")
|
| 64 |
+
return weights
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def _per_sample_group_balanced_smooth_l1(
|
| 68 |
+
prediction: torch.Tensor,
|
| 69 |
+
target: torch.Tensor,
|
| 70 |
+
mask: torch.Tensor,
|
| 71 |
+
*,
|
| 72 |
+
groups: ActionGroups,
|
| 73 |
+
timestep_weights: torch.Tensor | None = None,
|
| 74 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 75 |
+
"""Return group-balanced losses and a mask of nonempty samples."""
|
| 76 |
+
|
| 77 |
+
mask = _validate_action_tensors(prediction, target, mask)
|
| 78 |
+
groups = _validate_groups(groups, prediction.shape[-1])
|
| 79 |
+
weights = (
|
| 80 |
+
None
|
| 81 |
+
if timestep_weights is None
|
| 82 |
+
else _validate_timestep_weights(timestep_weights, prediction)
|
| 83 |
+
)
|
| 84 |
+
|
| 85 |
+
# Cached padded targets can contain NaN. Select a finite, detached value
|
| 86 |
+
# before Smooth L1 rather than masking a NaN loss after arithmetic.
|
| 87 |
+
safe_target = torch.where(mask, target, prediction.detach())
|
| 88 |
+
error = F.smooth_l1_loss(prediction, safe_target, reduction="none")
|
| 89 |
+
group_values, group_present = [], []
|
| 90 |
+
for start, stop in groups:
|
| 91 |
+
selected = mask[..., start:stop]
|
| 92 |
+
if weights is None:
|
| 93 |
+
denominator = selected.sum((1, 2))
|
| 94 |
+
numerator = error[..., start:stop].masked_fill(~selected, 0).sum((1, 2))
|
| 95 |
+
else:
|
| 96 |
+
# weights is [B,T,1] and broadcasts over the selected action group.
|
| 97 |
+
weighted_mask = weights * selected.to(dtype=error.dtype)
|
| 98 |
+
denominator = weighted_mask.sum((1, 2))
|
| 99 |
+
numerator = (error[..., start:stop] * weighted_mask).sum((1, 2))
|
| 100 |
+
# A normalized exponential weight sum can be below one (for example,
|
| 101 |
+
# one far-future scalar). Clamp only for numerical safety; clamping to
|
| 102 |
+
# one would silently shrink such a group's loss.
|
| 103 |
+
group_values.append(
|
| 104 |
+
numerator / denominator.clamp_min(torch.finfo(error.dtype).tiny)
|
| 105 |
+
)
|
| 106 |
+
group_present.append(denominator > 0)
|
| 107 |
+
|
| 108 |
+
present = torch.stack(group_present, dim=1)
|
| 109 |
+
per_sample = torch.stack(group_values, dim=1).sum(1) / present.sum(1).clamp_min(1)
|
| 110 |
+
valid = present.any(1)
|
| 111 |
+
return per_sample.masked_fill(~valid, 0), valid
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def per_sample_action_loss(
|
| 115 |
+
prediction: torch.Tensor,
|
| 116 |
+
target: torch.Tensor,
|
| 117 |
+
mask: torch.Tensor,
|
| 118 |
+
*,
|
| 119 |
+
groups: ActionGroups = ACTION_GROUPS,
|
| 120 |
+
) -> torch.Tensor:
|
| 121 |
+
"""Return the existing group-balanced Smooth L1 action loss as ``[B]``."""
|
| 122 |
+
|
| 123 |
+
values, valid = _per_sample_group_balanced_smooth_l1(
|
| 124 |
+
prediction, target, mask, groups=groups
|
| 125 |
+
)
|
| 126 |
+
if not valid.all():
|
| 127 |
+
raise ValueError("every action-loss sample must contain supervised actions")
|
| 128 |
+
return values
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def near_horizon_weights(
|
| 132 |
+
decision_seconds: torch.Tensor,
|
| 133 |
+
horizon: int,
|
| 134 |
+
*,
|
| 135 |
+
tau_seconds: float,
|
| 136 |
+
action_hz: float = 80.0,
|
| 137 |
+
dtype: torch.dtype = torch.float32,
|
| 138 |
+
) -> torch.Tensor:
|
| 139 |
+
"""Return ``exp(-(t_k-t_decision)/tau)`` weights shaped ``[B,T,1]``.
|
| 140 |
+
|
| 141 |
+
Negative delays are clamped to zero. A future/edit mask must still be
|
| 142 |
+
supplied to the loss; this function does not decide which actions are
|
| 143 |
+
committed.
|
| 144 |
+
"""
|
| 145 |
+
|
| 146 |
+
if decision_seconds.ndim != 1:
|
| 147 |
+
raise ValueError("decision_seconds must be [B]")
|
| 148 |
+
if horizon < 1:
|
| 149 |
+
raise ValueError("horizon must be positive")
|
| 150 |
+
if not math.isfinite(tau_seconds) or tau_seconds <= 0:
|
| 151 |
+
raise ValueError("tau_seconds must be finite and positive")
|
| 152 |
+
if not math.isfinite(action_hz) or action_hz <= 0:
|
| 153 |
+
raise ValueError("action_hz must be finite and positive")
|
| 154 |
+
decision = decision_seconds.to(dtype=torch.float64)
|
| 155 |
+
if not torch.isfinite(decision).all() or (decision < 0).any():
|
| 156 |
+
raise ValueError("decision_seconds must be finite and non-negative")
|
| 157 |
+
ticks = torch.arange(
|
| 158 |
+
horizon, device=decision.device, dtype=torch.float64
|
| 159 |
+
)[None, :, None] / action_hz
|
| 160 |
+
delay = (ticks - decision[:, None, None]).clamp_min(0)
|
| 161 |
+
return torch.exp(-delay / tau_seconds).to(dtype=dtype)
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def per_sample_near_horizon_action_loss(
|
| 165 |
+
prediction: torch.Tensor,
|
| 166 |
+
target: torch.Tensor,
|
| 167 |
+
mask: torch.Tensor,
|
| 168 |
+
decision_seconds: torch.Tensor,
|
| 169 |
+
*,
|
| 170 |
+
tau_seconds: float,
|
| 171 |
+
action_hz: float = 80.0,
|
| 172 |
+
groups: ActionGroups = ACTION_GROUPS,
|
| 173 |
+
) -> torch.Tensor:
|
| 174 |
+
"""Return near-horizon, group-balanced action losses as ``[B]``.
|
| 175 |
+
|
| 176 |
+
Each group is divided by its sum of valid temporal weights, so changing
|
| 177 |
+
``tau_seconds`` changes temporal emphasis without changing the loss scale.
|
| 178 |
+
"""
|
| 179 |
+
|
| 180 |
+
weights = near_horizon_weights(
|
| 181 |
+
decision_seconds,
|
| 182 |
+
prediction.shape[1],
|
| 183 |
+
tau_seconds=tau_seconds,
|
| 184 |
+
action_hz=action_hz,
|
| 185 |
+
dtype=prediction.dtype,
|
| 186 |
+
)
|
| 187 |
+
if weights.device != prediction.device:
|
| 188 |
+
weights = weights.to(prediction.device)
|
| 189 |
+
values, valid = _per_sample_group_balanced_smooth_l1(
|
| 190 |
+
prediction,
|
| 191 |
+
target,
|
| 192 |
+
mask,
|
| 193 |
+
groups=groups,
|
| 194 |
+
timestep_weights=weights,
|
| 195 |
+
)
|
| 196 |
+
if not valid.all():
|
| 197 |
+
raise ValueError("every near-horizon sample must contain supervised actions")
|
| 198 |
+
return values
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
def per_sample_motion_loss(
|
| 202 |
+
prediction: torch.Tensor,
|
| 203 |
+
target: torch.Tensor,
|
| 204 |
+
mask: torch.Tensor,
|
| 205 |
+
*,
|
| 206 |
+
groups: ActionGroups = ACTION_GROUPS,
|
| 207 |
+
timestep_weights: torch.Tensor | None = None,
|
| 208 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 209 |
+
"""Return adjacent-action motion losses and valid samples, both shaped ``[B]``.
|
| 210 |
+
|
| 211 |
+
A transition is supervised only when both endpoints are in ``mask``. With a
|
| 212 |
+
future edit mask this excludes the committed-to-first-editable boundary.
|
| 213 |
+
Optional ``timestep_weights`` has shape ``[B,T,1]``; transition ``k-1 -> k``
|
| 214 |
+
uses the weight at its later endpoint ``k``.
|
| 215 |
+
"""
|
| 216 |
+
|
| 217 |
+
mask = _validate_action_tensors(prediction, target, mask)
|
| 218 |
+
weights = (
|
| 219 |
+
None
|
| 220 |
+
if timestep_weights is None
|
| 221 |
+
else _validate_timestep_weights(timestep_weights, prediction)[:, 1:]
|
| 222 |
+
)
|
| 223 |
+
safe_target = torch.where(mask, target, prediction.detach())
|
| 224 |
+
predicted_motion = prediction[:, 1:] - prediction[:, :-1]
|
| 225 |
+
target_motion = safe_target[:, 1:] - safe_target[:, :-1]
|
| 226 |
+
pair_mask = mask[:, 1:] & mask[:, :-1]
|
| 227 |
+
return _per_sample_group_balanced_smooth_l1(
|
| 228 |
+
predicted_motion,
|
| 229 |
+
target_motion,
|
| 230 |
+
pair_mask,
|
| 231 |
+
groups=groups,
|
| 232 |
+
timestep_weights=weights,
|
| 233 |
+
)
|
model/tachin_paired30_cache_20260923/code/src/tactile_editor/action_output.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Model-independent action output contract for Table 1 evaluation."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from dataclasses import dataclass, field
|
| 6 |
+
from typing import Any
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
|
| 10 |
+
from .action_layout import (
|
| 11 |
+
ACTION_DIM,
|
| 12 |
+
ARTICULATION_GROUPS,
|
| 13 |
+
CHUNK_SIZE,
|
| 14 |
+
DEFAULT_ACTION_LAYOUT,
|
| 15 |
+
FULL_ACTION_GROUPS,
|
| 16 |
+
)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@dataclass(frozen=True)
|
| 20 |
+
class NormalizedActionOutput:
|
| 21 |
+
"""A model prediction in the frozen VLA normalized action space.
|
| 22 |
+
|
| 23 |
+
Every baseline row (native VLA, tactile editor, or adapted model) can be
|
| 24 |
+
exported in this format before denormalization and physical metric scoring.
|
| 25 |
+
``valid_mask`` describes available hand/action values; padding is rejected
|
| 26 |
+
at construction time so it cannot enter losses or Table 1 metrics.
|
| 27 |
+
"""
|
| 28 |
+
|
| 29 |
+
actions: torch.Tensor
|
| 30 |
+
valid_mask: torch.Tensor
|
| 31 |
+
model_id: str
|
| 32 |
+
normalization_id: str
|
| 33 |
+
metadata: dict[str, Any] = field(default_factory=dict)
|
| 34 |
+
|
| 35 |
+
def __post_init__(self) -> None:
|
| 36 |
+
if self.actions.ndim != 3 or tuple(self.actions.shape[1:]) != (
|
| 37 |
+
CHUNK_SIZE,
|
| 38 |
+
ACTION_DIM,
|
| 39 |
+
):
|
| 40 |
+
raise ValueError("actions must be [B,16,192]")
|
| 41 |
+
if self.valid_mask.shape != self.actions.shape:
|
| 42 |
+
raise ValueError("valid_mask must match actions")
|
| 43 |
+
if not self.actions.is_floating_point() or not torch.isfinite(self.actions).all():
|
| 44 |
+
raise ValueError("actions must be finite floating-point values")
|
| 45 |
+
if self.valid_mask[..., 138:].any():
|
| 46 |
+
raise ValueError("padding dimensions 138:192 must be invalid")
|
| 47 |
+
if not self.model_id or not self.normalization_id:
|
| 48 |
+
raise ValueError("model_id and normalization_id are required")
|
| 49 |
+
|
| 50 |
+
@property
|
| 51 |
+
def physical_mask(self) -> torch.Tensor:
|
| 52 |
+
return self.valid_mask.bool() & DEFAULT_ACTION_LAYOUT.physical_mask(
|
| 53 |
+
tuple(self.valid_mask.shape), device=self.valid_mask.device
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
def editable(self, *, articulation_only: bool = False) -> torch.Tensor:
|
| 57 |
+
groups = ARTICULATION_GROUPS if articulation_only else FULL_ACTION_GROUPS
|
| 58 |
+
return self.valid_mask.bool() & DEFAULT_ACTION_LAYOUT.physical_mask(
|
| 59 |
+
tuple(self.valid_mask.shape),
|
| 60 |
+
device=self.valid_mask.device,
|
| 61 |
+
groups=groups,
|
| 62 |
+
)
|
| 63 |
+
|
| 64 |
+
def denormalize(self, mean: torch.Tensor, std: torch.Tensor) -> torch.Tensor:
|
| 65 |
+
if tuple(mean.shape) != (ACTION_DIM,) or tuple(std.shape) != (ACTION_DIM,):
|
| 66 |
+
raise ValueError("normalization statistics must be [192]")
|
| 67 |
+
if not torch.isfinite(mean).all() or not torch.isfinite(std).all() or (std <= 0).any():
|
| 68 |
+
raise ValueError("normalization statistics must be finite and positive")
|
| 69 |
+
mean = mean.to(device=self.actions.device, dtype=self.actions.dtype)
|
| 70 |
+
std = std.to(device=self.actions.device, dtype=self.actions.dtype)
|
| 71 |
+
return self.actions * std + mean
|
model/tachin_paired30_cache_20260923/code/src/tactile_editor/architecture_ablation.py
ADDED
|
@@ -0,0 +1,423 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Stage-2-only tactile conditioning architecture ablations.
|
| 2 |
+
|
| 3 |
+
The calibrated action is an immutable Stage-1 anchor. These research models
|
| 4 |
+
change only how frozen tactile tokens condition an incremental residual.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from dataclasses import dataclass
|
| 10 |
+
import math
|
| 11 |
+
|
| 12 |
+
import torch
|
| 13 |
+
from torch import nn
|
| 14 |
+
|
| 15 |
+
from .touch_study import IncrementalTouchEditor, StudySetting
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
ARCHITECTURES = ("e0_cross_attention", "e1_adaln", "e2_basis", "e3_centered_basis")
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@dataclass(frozen=True)
|
| 22 |
+
class ArchitectureSpec:
|
| 23 |
+
name: str
|
| 24 |
+
conditioning: str
|
| 25 |
+
output: str
|
| 26 |
+
centered_coefficients: bool = False
|
| 27 |
+
residual_bases: int = 0
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
SPECS = {
|
| 31 |
+
"e0_cross_attention": ArchitectureSpec(
|
| 32 |
+
"e0_cross_attention", "gated_action_to_touch_cross_attention", "direct_delta"
|
| 33 |
+
),
|
| 34 |
+
"e1_adaln": ArchitectureSpec(
|
| 35 |
+
"e1_adaln", "adaln_zero", "direct_delta"
|
| 36 |
+
),
|
| 37 |
+
"e2_basis": ArchitectureSpec(
|
| 38 |
+
"e2_basis", "adaln_zero", "tactile_weighted_residual_bases", residual_bases=8
|
| 39 |
+
),
|
| 40 |
+
"e3_centered_basis": ArchitectureSpec(
|
| 41 |
+
"e3_centered_basis",
|
| 42 |
+
"adaln_zero",
|
| 43 |
+
"tactile_weighted_residual_bases",
|
| 44 |
+
centered_coefficients=True,
|
| 45 |
+
residual_bases=8,
|
| 46 |
+
),
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def cross_video_phase_matched_donors(
|
| 51 |
+
videos: list[str],
|
| 52 |
+
elapsed: torch.Tensor,
|
| 53 |
+
hand_valid: torch.Tensor,
|
| 54 |
+
*,
|
| 55 |
+
seed: int = 20260907,
|
| 56 |
+
) -> torch.Tensor:
|
| 57 |
+
"""Build a deterministic one-to-one tactile shuffle within phase/validity groups.
|
| 58 |
+
|
| 59 |
+
Grouped cyclic rotation is a true permutation and guarantees that the donor
|
| 60 |
+
source video differs when no video owns more than half of a group.
|
| 61 |
+
"""
|
| 62 |
+
if len(videos) != len(elapsed) or len(videos) != len(hand_valid):
|
| 63 |
+
raise ValueError("video, elapsed, and hand_valid lengths must match")
|
| 64 |
+
generator = torch.Generator().manual_seed(seed)
|
| 65 |
+
phases = torch.round(elapsed.detach().cpu().float() * 30).long()
|
| 66 |
+
validity = hand_valid.detach().cpu().bool()
|
| 67 |
+
donors = torch.full((len(videos),), -1, dtype=torch.long)
|
| 68 |
+
keys = sorted(
|
| 69 |
+
{
|
| 70 |
+
(int(phases[i]), bool(validity[i, 0]), bool(validity[i, 1]))
|
| 71 |
+
for i in range(len(videos))
|
| 72 |
+
}
|
| 73 |
+
)
|
| 74 |
+
for key in keys:
|
| 75 |
+
indices = [
|
| 76 |
+
i
|
| 77 |
+
for i in range(len(videos))
|
| 78 |
+
if (int(phases[i]), bool(validity[i, 0]), bool(validity[i, 1])) == key
|
| 79 |
+
]
|
| 80 |
+
by_video: dict[str, list[int]] = {}
|
| 81 |
+
for index in indices:
|
| 82 |
+
by_video.setdefault(videos[index], []).append(index)
|
| 83 |
+
maximum = max(map(len, by_video.values()))
|
| 84 |
+
if maximum * 2 > len(indices):
|
| 85 |
+
raise ValueError(
|
| 86 |
+
f"cannot construct cross-video permutation for group {key}: "
|
| 87 |
+
f"largest video has {maximum}/{len(indices)} samples"
|
| 88 |
+
)
|
| 89 |
+
names = sorted(by_video)
|
| 90 |
+
names = [names[i] for i in torch.randperm(len(names), generator=generator)]
|
| 91 |
+
ordered = []
|
| 92 |
+
for name in names:
|
| 93 |
+
rows = by_video[name]
|
| 94 |
+
order = torch.randperm(len(rows), generator=generator).tolist()
|
| 95 |
+
ordered.extend(rows[i] for i in order)
|
| 96 |
+
rotated = ordered[maximum:] + ordered[:maximum]
|
| 97 |
+
if any(videos[source] == videos[donor] for source, donor in zip(ordered, rotated)):
|
| 98 |
+
raise AssertionError("grouped rotation failed to change source video")
|
| 99 |
+
for source, donor in zip(ordered, rotated):
|
| 100 |
+
donors[source] = donor
|
| 101 |
+
if (donors < 0).any() or len(torch.unique(donors)) != len(donors):
|
| 102 |
+
raise AssertionError("donor mapping must be a complete permutation")
|
| 103 |
+
return donors
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def within_video_phase_matched_donors(
|
| 107 |
+
videos: list[str],
|
| 108 |
+
elapsed: torch.Tensor,
|
| 109 |
+
hand_valid: torch.Tensor,
|
| 110 |
+
*,
|
| 111 |
+
seed: int = 20260907,
|
| 112 |
+
) -> torch.Tensor:
|
| 113 |
+
"""Shuffle tactile samples inside one recording and decision phase.
|
| 114 |
+
|
| 115 |
+
This control preserves recording identity, decision phase, and the two-hand
|
| 116 |
+
validity pattern. It breaks the local action-touch pairing without creating
|
| 117 |
+
the cross-recording context shift of the primary shuffled control.
|
| 118 |
+
"""
|
| 119 |
+
|
| 120 |
+
if len(videos) != len(elapsed) or len(videos) != len(hand_valid):
|
| 121 |
+
raise ValueError("video, elapsed, and hand_valid lengths must match")
|
| 122 |
+
generator = torch.Generator().manual_seed(seed)
|
| 123 |
+
phases = torch.round(elapsed.detach().cpu().float() * 30).long()
|
| 124 |
+
validity = hand_valid.detach().cpu().bool()
|
| 125 |
+
donors = torch.full((len(videos),), -1, dtype=torch.long)
|
| 126 |
+
groups: dict[tuple[str, int, bool, bool], list[int]] = {}
|
| 127 |
+
for index, video in enumerate(videos):
|
| 128 |
+
key = (
|
| 129 |
+
video,
|
| 130 |
+
int(phases[index]),
|
| 131 |
+
bool(validity[index, 0]),
|
| 132 |
+
bool(validity[index, 1]),
|
| 133 |
+
)
|
| 134 |
+
groups.setdefault(key, []).append(index)
|
| 135 |
+
for key, indices in sorted(groups.items()):
|
| 136 |
+
if len(indices) < 2:
|
| 137 |
+
raise ValueError(f"cannot shuffle singleton within-video group {key}")
|
| 138 |
+
order = torch.randperm(len(indices), generator=generator).tolist()
|
| 139 |
+
ordered = [indices[position] for position in order]
|
| 140 |
+
rotated = ordered[1:] + ordered[:1]
|
| 141 |
+
for source, donor in zip(ordered, rotated):
|
| 142 |
+
donors[source] = donor
|
| 143 |
+
if (donors < 0).any() or len(torch.unique(donors)) != len(donors):
|
| 144 |
+
raise AssertionError("within-video donor mapping must be a complete permutation")
|
| 145 |
+
if any(videos[i] != videos[int(donors[i])] for i in range(len(videos))):
|
| 146 |
+
raise AssertionError("within-video donor changed recording")
|
| 147 |
+
if (donors == torch.arange(len(donors))).any():
|
| 148 |
+
raise AssertionError("within-video donor mapping must change every sample")
|
| 149 |
+
return donors
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def tactile_contact_statistics(
|
| 153 |
+
touch_pressure: torch.Tensor,
|
| 154 |
+
touch_valid: torch.Tensor,
|
| 155 |
+
pressure_masks: torch.Tensor,
|
| 156 |
+
) -> dict[str, torch.Tensor]:
|
| 157 |
+
"""Compute causal per-sample contact scores from two latest raw 30 Hz frames."""
|
| 158 |
+
if touch_pressure.ndim != 5 or tuple(touch_pressure.shape[1:]) != (8, 2, 21, 21):
|
| 159 |
+
raise ValueError("touch_pressure must be [N,8,2,21,21]")
|
| 160 |
+
if tuple(touch_valid.shape) != tuple(touch_pressure.shape[:3]):
|
| 161 |
+
raise ValueError("touch_valid must be [N,8,2]")
|
| 162 |
+
if tuple(pressure_masks.shape) != (2, 21, 21):
|
| 163 |
+
raise ValueError("pressure_masks must be [2,21,21]")
|
| 164 |
+
pressure = torch.nan_to_num(touch_pressure.detach().cpu().float()).clone()
|
| 165 |
+
pressure[:, :, 0] = pressure[:, :, 0].flip(-1)
|
| 166 |
+
masks = pressure_masks.detach().cpu().bool()
|
| 167 |
+
cells = torch.stack(
|
| 168 |
+
[pressure[:, :, hand, masks[hand]] for hand in range(2)], dim=2
|
| 169 |
+
)
|
| 170 |
+
scores = cells.topk(5, dim=-1).values.mean(-1)
|
| 171 |
+
pair_valid = touch_valid.detach().cpu().bool()[:, -2:].all(1)
|
| 172 |
+
current_valid = touch_valid.detach().cpu().bool()[:, -1]
|
| 173 |
+
current = scores[:, -1]
|
| 174 |
+
previous = scores[:, -2]
|
| 175 |
+
return {
|
| 176 |
+
"current_score_per_hand": current,
|
| 177 |
+
"previous_score_per_hand": previous,
|
| 178 |
+
"current_valid_per_hand": current_valid,
|
| 179 |
+
"pair_valid_per_hand": pair_valid,
|
| 180 |
+
}
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def tactile_event_change(
|
| 184 |
+
touch_pressure: torch.Tensor,
|
| 185 |
+
touch_valid: torch.Tensor,
|
| 186 |
+
pressure_masks: torch.Tensor,
|
| 187 |
+
) -> torch.Tensor:
|
| 188 |
+
"""Mean absolute pressure change over valid cells and hands at the decision."""
|
| 189 |
+
|
| 190 |
+
if touch_pressure.ndim != 5 or tuple(touch_pressure.shape[1:]) != (8, 2, 21, 21):
|
| 191 |
+
raise ValueError("touch_pressure must be [N,8,2,21,21]")
|
| 192 |
+
if tuple(touch_valid.shape) != tuple(touch_pressure.shape[:3]):
|
| 193 |
+
raise ValueError("touch_valid must be [N,8,2]")
|
| 194 |
+
pressure = torch.nan_to_num(touch_pressure.detach().cpu().float()).clone()
|
| 195 |
+
pressure[:, :, 0] = pressure[:, :, 0].flip(-1)
|
| 196 |
+
masks = pressure_masks.detach().cpu().bool()
|
| 197 |
+
per_hand = []
|
| 198 |
+
for hand in range(2):
|
| 199 |
+
cells = pressure[:, -2:, hand, masks[hand]]
|
| 200 |
+
per_hand.append((cells[:, 1] - cells[:, 0]).abs().mean(-1))
|
| 201 |
+
change = torch.stack(per_hand, dim=1)
|
| 202 |
+
valid = touch_valid.detach().cpu().bool()[:, -2:].all(1)
|
| 203 |
+
return (change * valid).sum(1) / valid.sum(1).clamp_min(1)
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
def contact_slice_masks(
|
| 207 |
+
statistics: dict[str, torch.Tensor],
|
| 208 |
+
event_change: torch.Tensor,
|
| 209 |
+
*,
|
| 210 |
+
high_contact_threshold: float,
|
| 211 |
+
pressure_change_threshold: float,
|
| 212 |
+
contact_threshold: float = 0.1,
|
| 213 |
+
) -> dict[str, torch.Tensor]:
|
| 214 |
+
"""Return the five overlapping evaluation slices used by the ablation."""
|
| 215 |
+
current = statistics["current_score_per_hand"]
|
| 216 |
+
previous = statistics["previous_score_per_hand"]
|
| 217 |
+
current_valid = statistics["current_valid_per_hand"]
|
| 218 |
+
pair_valid = statistics["pair_valid_per_hand"]
|
| 219 |
+
now_contact = current >= contact_threshold
|
| 220 |
+
previous_contact = previous >= contact_threshold
|
| 221 |
+
transition = ((now_contact != previous_contact) & pair_valid).any(1)
|
| 222 |
+
changing = event_change.detach().cpu().float() >= pressure_change_threshold
|
| 223 |
+
stable_contact = (
|
| 224 |
+
(now_contact & previous_contact & pair_valid).any(1)
|
| 225 |
+
& ~transition
|
| 226 |
+
& ~changing
|
| 227 |
+
)
|
| 228 |
+
low = pair_valid.any(1) & ~(
|
| 229 |
+
(now_contact | previous_contact) & pair_valid
|
| 230 |
+
).any(1)
|
| 231 |
+
high = (
|
| 232 |
+
(current >= high_contact_threshold) & current_valid
|
| 233 |
+
).any(1)
|
| 234 |
+
return {
|
| 235 |
+
"high_contact": high,
|
| 236 |
+
"contact_transition": transition,
|
| 237 |
+
"pressure_changing": changing,
|
| 238 |
+
"stable_contact": stable_contact,
|
| 239 |
+
"low_or_no_contact": low,
|
| 240 |
+
}
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
def _modulate(x: torch.Tensor, shift: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
|
| 244 |
+
return x * (1 + scale[:, None]) + shift[:, None]
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
class AdaLNZeroBlock(nn.Module):
|
| 248 |
+
"""One pre-norm Transformer block with zero-initialized tactile modulation."""
|
| 249 |
+
|
| 250 |
+
def __init__(self, hidden: int = 128, heads: int = 4, dropout: float = 0.1):
|
| 251 |
+
super().__init__()
|
| 252 |
+
self.norm_attention = nn.LayerNorm(hidden, elementwise_affine=False)
|
| 253 |
+
self.attention = nn.MultiheadAttention(
|
| 254 |
+
hidden, heads, dropout=dropout, batch_first=True
|
| 255 |
+
)
|
| 256 |
+
self.norm_mlp = nn.LayerNorm(hidden, elementwise_affine=False)
|
| 257 |
+
self.mlp = nn.Sequential(
|
| 258 |
+
nn.Linear(hidden, hidden * 4),
|
| 259 |
+
nn.GELU(),
|
| 260 |
+
nn.Dropout(dropout),
|
| 261 |
+
nn.Linear(hidden * 4, hidden),
|
| 262 |
+
nn.Dropout(dropout),
|
| 263 |
+
)
|
| 264 |
+
self.modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden, hidden * 6))
|
| 265 |
+
nn.init.zeros_(self.modulation[-1].weight)
|
| 266 |
+
nn.init.zeros_(self.modulation[-1].bias)
|
| 267 |
+
|
| 268 |
+
def forward(self, x: torch.Tensor, condition: torch.Tensor) -> torch.Tensor:
|
| 269 |
+
shift_attn, scale_attn, gate_attn, shift_mlp, scale_mlp, gate_mlp = (
|
| 270 |
+
self.modulation(condition).chunk(6, dim=-1)
|
| 271 |
+
)
|
| 272 |
+
q = _modulate(self.norm_attention(x), shift_attn, scale_attn)
|
| 273 |
+
attended = self.attention(q, q, q, need_weights=False)[0]
|
| 274 |
+
x = x + gate_attn[:, None] * attended
|
| 275 |
+
feedforward = self.mlp(_modulate(self.norm_mlp(x), shift_mlp, scale_mlp))
|
| 276 |
+
return x + gate_mlp[:, None] * feedforward
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
class AdaLNZeroTouchEditor(nn.Module):
|
| 280 |
+
"""AdaLN-Zero direct-head or tactile-weighted-basis editor.
|
| 281 |
+
|
| 282 |
+
E2/E3 intentionally have no action-only delta output. Their residual can
|
| 283 |
+
only be formed by tactile coefficients weighting action-trunk bases.
|
| 284 |
+
"""
|
| 285 |
+
|
| 286 |
+
def __init__(
|
| 287 |
+
self,
|
| 288 |
+
action_mean: torch.Tensor,
|
| 289 |
+
action_std: torch.Tensor,
|
| 290 |
+
architecture: str,
|
| 291 |
+
*,
|
| 292 |
+
hidden: int = 128,
|
| 293 |
+
residual_bases: int = 8,
|
| 294 |
+
dropout: float = 0.1,
|
| 295 |
+
):
|
| 296 |
+
super().__init__()
|
| 297 |
+
if architecture not in ("e1_adaln", "e2_basis", "e3_centered_basis"):
|
| 298 |
+
raise ValueError(f"unsupported AdaLN architecture: {architecture}")
|
| 299 |
+
if residual_bases < 1:
|
| 300 |
+
raise ValueError("residual_bases must be positive")
|
| 301 |
+
self.architecture = architecture
|
| 302 |
+
self.hidden = hidden
|
| 303 |
+
self.residual_bases = residual_bases
|
| 304 |
+
self.centered_coefficients = architecture == "e3_centered_basis"
|
| 305 |
+
self.uses_basis_output = architecture in ("e2_basis", "e3_centered_basis")
|
| 306 |
+
self.register_buffer("action_mean", action_mean)
|
| 307 |
+
self.register_buffer("action_std", action_std.clamp_min(1e-3))
|
| 308 |
+
self.action_projection = nn.Linear(192, hidden)
|
| 309 |
+
self.position = nn.Parameter(torch.randn(1, 16, hidden) * 0.02)
|
| 310 |
+
self.time_projection = nn.Linear(1, hidden)
|
| 311 |
+
# Concatenation retains the fixed left/right identity of the two 64-D tokens.
|
| 312 |
+
self.touch_condition = nn.Linear(128, hidden)
|
| 313 |
+
self.trunk = AdaLNZeroBlock(hidden, heads=4, dropout=dropout)
|
| 314 |
+
|
| 315 |
+
if self.uses_basis_output:
|
| 316 |
+
self.basis_head = nn.Linear(hidden, residual_bases * 192)
|
| 317 |
+
nn.init.zeros_(self.basis_head.weight)
|
| 318 |
+
nn.init.zeros_(self.basis_head.bias)
|
| 319 |
+
self.coefficient_head = nn.Linear(
|
| 320 |
+
128, residual_bases, bias=not self.centered_coefficients
|
| 321 |
+
)
|
| 322 |
+
else:
|
| 323 |
+
self.output = nn.Linear(hidden, 192)
|
| 324 |
+
nn.init.zeros_(self.output.weight)
|
| 325 |
+
nn.init.zeros_(self.output.bias)
|
| 326 |
+
|
| 327 |
+
@staticmethod
|
| 328 |
+
def _masked_dual_hand(tokens: torch.Tensor, hand_valid: torch.Tensor) -> torch.Tensor:
|
| 329 |
+
if tokens.ndim != 3 or tuple(tokens.shape[1:]) != (2, 64):
|
| 330 |
+
raise ValueError(f"tokens must be [B,2,64], got {tuple(tokens.shape)}")
|
| 331 |
+
if tuple(hand_valid.shape) != tuple(tokens.shape[:2]):
|
| 332 |
+
raise ValueError("hand_valid must match tokens' first two dimensions")
|
| 333 |
+
masked = tokens * hand_valid[..., None].to(tokens.dtype)
|
| 334 |
+
return masked.flatten(1)
|
| 335 |
+
|
| 336 |
+
def tactile_coefficients(
|
| 337 |
+
self,
|
| 338 |
+
tokens: torch.Tensor,
|
| 339 |
+
hand_valid: torch.Tensor,
|
| 340 |
+
mean_tokens: torch.Tensor,
|
| 341 |
+
) -> torch.Tensor:
|
| 342 |
+
if not self.uses_basis_output:
|
| 343 |
+
raise RuntimeError("direct-head E1 has no residual-basis coefficients")
|
| 344 |
+
coefficient_tokens = tokens
|
| 345 |
+
if self.centered_coefficients:
|
| 346 |
+
coefficient_tokens = tokens - mean_tokens.expand_as(tokens)
|
| 347 |
+
return self.coefficient_head(
|
| 348 |
+
self._masked_dual_hand(coefficient_tokens, hand_valid)
|
| 349 |
+
)
|
| 350 |
+
|
| 351 |
+
def forward(
|
| 352 |
+
self,
|
| 353 |
+
anchor: torch.Tensor,
|
| 354 |
+
mask: torch.Tensor,
|
| 355 |
+
tokens: torch.Tensor,
|
| 356 |
+
hand_valid: torch.Tensor,
|
| 357 |
+
elapsed: torch.Tensor,
|
| 358 |
+
predicted_tokens: torch.Tensor,
|
| 359 |
+
mean_tokens: torch.Tensor,
|
| 360 |
+
*,
|
| 361 |
+
coefficient_override: torch.Tensor | None = None,
|
| 362 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 363 |
+
del predicted_tokens # Predicted innovation is deliberately outside this ablation.
|
| 364 |
+
action = self.action_projection(
|
| 365 |
+
(anchor.detach() - self.action_mean) / self.action_std
|
| 366 |
+
)
|
| 367 |
+
action = (
|
| 368 |
+
action
|
| 369 |
+
+ self.position
|
| 370 |
+
+ self.time_projection(elapsed.float()[:, None, None] * 10)
|
| 371 |
+
)
|
| 372 |
+
tactile = self._masked_dual_hand(tokens, hand_valid)
|
| 373 |
+
hidden = self.trunk(action, self.touch_condition(tactile))
|
| 374 |
+
|
| 375 |
+
if self.uses_basis_output:
|
| 376 |
+
bases = self.basis_head(hidden).reshape(
|
| 377 |
+
len(anchor), 16, self.residual_bases, 192
|
| 378 |
+
)
|
| 379 |
+
bases = bases.permute(0, 2, 1, 3)
|
| 380 |
+
coefficients = (
|
| 381 |
+
self.tactile_coefficients(tokens, hand_valid, mean_tokens)
|
| 382 |
+
if coefficient_override is None
|
| 383 |
+
else coefficient_override
|
| 384 |
+
)
|
| 385 |
+
expected = (len(anchor), self.residual_bases)
|
| 386 |
+
if tuple(coefficients.shape) != expected:
|
| 387 |
+
raise ValueError(
|
| 388 |
+
f"coefficients must have shape {expected}, got {tuple(coefficients.shape)}"
|
| 389 |
+
)
|
| 390 |
+
# Explicit broadcast avoids a version-specific compiled einsum
|
| 391 |
+
# backward while preserving the exact weighted-basis definition.
|
| 392 |
+
raw_delta = (coefficients[:, :, None, None] * bases).sum(1)
|
| 393 |
+
raw_delta = raw_delta / math.sqrt(self.residual_bases)
|
| 394 |
+
else:
|
| 395 |
+
raw_delta = self.output(hidden)
|
| 396 |
+
|
| 397 |
+
valid_sample = hand_valid.any(1)[:, None, None]
|
| 398 |
+
delta = raw_delta * mask.to(raw_delta.dtype) * valid_sample.to(raw_delta.dtype)
|
| 399 |
+
return anchor.detach() + delta, delta
|
| 400 |
+
|
| 401 |
+
|
| 402 |
+
def build_architecture_editor(
|
| 403 |
+
architecture: str,
|
| 404 |
+
action_mean: torch.Tensor,
|
| 405 |
+
action_std: torch.Tensor,
|
| 406 |
+
*,
|
| 407 |
+
hidden: int = 128,
|
| 408 |
+
residual_bases: int = 8,
|
| 409 |
+
) -> nn.Module:
|
| 410 |
+
"""Construct an ablation editor without changing the production defaults."""
|
| 411 |
+
if architecture not in ARCHITECTURES:
|
| 412 |
+
raise ValueError(f"unknown architecture: {architecture}")
|
| 413 |
+
if architecture == "e0_cross_attention":
|
| 414 |
+
return IncrementalTouchEditor(
|
| 415 |
+
action_mean, action_std, StudySetting("frozen"), hidden=hidden
|
| 416 |
+
)
|
| 417 |
+
return AdaLNZeroTouchEditor(
|
| 418 |
+
action_mean,
|
| 419 |
+
action_std,
|
| 420 |
+
architecture,
|
| 421 |
+
hidden=hidden,
|
| 422 |
+
residual_bases=residual_bases,
|
| 423 |
+
)
|
model/tachin_paired30_cache_20260923/code/src/tactile_editor/auxiliary_objectives.py
ADDED
|
@@ -0,0 +1,264 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Leakage-safe auxiliary objectives for the direct tactile residual editor.
|
| 2 |
+
|
| 3 |
+
These modules are training-only. They consume causal features already emitted
|
| 4 |
+
by ``DirectTactileResidualEditor``; neither module adds an inference-time input
|
| 5 |
+
to the editor.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import math
|
| 11 |
+
|
| 12 |
+
import torch
|
| 13 |
+
from torch import nn
|
| 14 |
+
from torch.nn import functional as F
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
LEFT_ARTICULATION = (6, 69)
|
| 18 |
+
RIGHT_ARTICULATION = (75, 138)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def active_hands_from_edit_mask(edit_mask: torch.Tensor) -> torch.Tensor:
|
| 22 |
+
"""Return ``[B,2]`` left/right articulation supervision flags."""
|
| 23 |
+
|
| 24 |
+
if edit_mask.ndim != 3 or edit_mask.shape[-1] < RIGHT_ARTICULATION[1]:
|
| 25 |
+
raise ValueError("edit_mask must be [B,T,A] with A >= 138")
|
| 26 |
+
left = (
|
| 27 |
+
edit_mask[..., LEFT_ARTICULATION[0] : LEFT_ARTICULATION[1]].bool().any((1, 2))
|
| 28 |
+
)
|
| 29 |
+
right = (
|
| 30 |
+
edit_mask[..., RIGHT_ARTICULATION[0] : RIGHT_ARTICULATION[1]].bool().any((1, 2))
|
| 31 |
+
)
|
| 32 |
+
return torch.stack((left, right), dim=1)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
class ActiveHandTouchResidualInfoNCE(nn.Module):
|
| 36 |
+
"""Align causal touch with ``target_action - anchor`` using hard groups.
|
| 37 |
+
|
| 38 |
+
A negative participates only when it has the same tactile decision tick and
|
| 39 |
+
active-hand code as the query. Rows in a group of size one are omitted. The
|
| 40 |
+
tactile projection receives only features from hands whose articulation is
|
| 41 |
+
editable, which prevents the inactive sensor from becoming a shortcut.
|
| 42 |
+
"""
|
| 43 |
+
|
| 44 |
+
def __init__(
|
| 45 |
+
self,
|
| 46 |
+
*,
|
| 47 |
+
tactile_feature_dim: int = 128,
|
| 48 |
+
action_horizon: int = 16,
|
| 49 |
+
hidden_dim: int = 128,
|
| 50 |
+
embedding_dim: int = 64,
|
| 51 |
+
temperature: float = 0.1,
|
| 52 |
+
tactile_hz: float = 30.0,
|
| 53 |
+
) -> None:
|
| 54 |
+
super().__init__()
|
| 55 |
+
if tactile_feature_dim < 1 or action_horizon < 1:
|
| 56 |
+
raise ValueError("feature dimension and action horizon must be positive")
|
| 57 |
+
if hidden_dim < 1 or embedding_dim < 1:
|
| 58 |
+
raise ValueError("hidden and embedding dimensions must be positive")
|
| 59 |
+
if not math.isfinite(temperature) or temperature <= 0:
|
| 60 |
+
raise ValueError("temperature must be finite and positive")
|
| 61 |
+
if not math.isfinite(tactile_hz) or tactile_hz <= 0:
|
| 62 |
+
raise ValueError("tactile_hz must be finite and positive")
|
| 63 |
+
articulation_dim = (
|
| 64 |
+
LEFT_ARTICULATION[1]
|
| 65 |
+
- LEFT_ARTICULATION[0]
|
| 66 |
+
+ RIGHT_ARTICULATION[1]
|
| 67 |
+
- RIGHT_ARTICULATION[0]
|
| 68 |
+
)
|
| 69 |
+
self.tactile_feature_dim = int(tactile_feature_dim)
|
| 70 |
+
self.action_horizon = int(action_horizon)
|
| 71 |
+
self.temperature = float(temperature)
|
| 72 |
+
self.tactile_hz = float(tactile_hz)
|
| 73 |
+
self.touch_projection = nn.Sequential(
|
| 74 |
+
nn.LayerNorm(2 * tactile_feature_dim),
|
| 75 |
+
nn.Linear(2 * tactile_feature_dim, hidden_dim),
|
| 76 |
+
nn.GELU(),
|
| 77 |
+
nn.Linear(hidden_dim, embedding_dim),
|
| 78 |
+
)
|
| 79 |
+
self.residual_projection = nn.Sequential(
|
| 80 |
+
nn.LayerNorm(action_horizon * articulation_dim),
|
| 81 |
+
nn.Linear(action_horizon * articulation_dim, hidden_dim),
|
| 82 |
+
nn.GELU(),
|
| 83 |
+
nn.Linear(hidden_dim, embedding_dim),
|
| 84 |
+
)
|
| 85 |
+
|
| 86 |
+
def forward(
|
| 87 |
+
self,
|
| 88 |
+
tactile_features: torch.Tensor,
|
| 89 |
+
target_action: torch.Tensor,
|
| 90 |
+
anchor: torch.Tensor,
|
| 91 |
+
edit_mask: torch.Tensor,
|
| 92 |
+
decision_seconds: torch.Tensor,
|
| 93 |
+
) -> torch.Tensor:
|
| 94 |
+
"""Return sample-weighted symmetric InfoNCE over eligible groups."""
|
| 95 |
+
|
| 96 |
+
batch = len(tactile_features)
|
| 97 |
+
if tactile_features.shape != (batch, 2, self.tactile_feature_dim):
|
| 98 |
+
raise ValueError(
|
| 99 |
+
f"tactile_features must be [B,2,{self.tactile_feature_dim}]"
|
| 100 |
+
)
|
| 101 |
+
if target_action.shape != anchor.shape or anchor.ndim != 3:
|
| 102 |
+
raise ValueError(
|
| 103 |
+
"target_action and anchor must have matching [B,T,A] shapes"
|
| 104 |
+
)
|
| 105 |
+
if anchor.shape[:2] != (batch, self.action_horizon):
|
| 106 |
+
raise ValueError(
|
| 107 |
+
f"target_action and anchor must be [B,{self.action_horizon},A]"
|
| 108 |
+
)
|
| 109 |
+
if edit_mask.shape != anchor.shape:
|
| 110 |
+
raise ValueError("edit_mask must match the action tensors")
|
| 111 |
+
if decision_seconds.shape != (batch,):
|
| 112 |
+
raise ValueError("decision_seconds must be [B]")
|
| 113 |
+
if not torch.isfinite(decision_seconds).all():
|
| 114 |
+
raise ValueError("decision_seconds must be finite")
|
| 115 |
+
|
| 116 |
+
active_hands = active_hands_from_edit_mask(edit_mask)
|
| 117 |
+
active_code = active_hands[:, 0].long() + 2 * active_hands[:, 1].long()
|
| 118 |
+
|
| 119 |
+
touch = tactile_features * active_hands[..., None].to(tactile_features.dtype)
|
| 120 |
+
touch_embedding = F.normalize(self.touch_projection(touch.flatten(1)), dim=-1)
|
| 121 |
+
|
| 122 |
+
left_mask = edit_mask[..., LEFT_ARTICULATION[0] : LEFT_ARTICULATION[1]].bool()
|
| 123 |
+
right_mask = edit_mask[
|
| 124 |
+
..., RIGHT_ARTICULATION[0] : RIGHT_ARTICULATION[1]
|
| 125 |
+
].bool()
|
| 126 |
+
articulation_mask = torch.cat((left_mask, right_mask), dim=-1)
|
| 127 |
+
if not torch.isfinite(anchor).all():
|
| 128 |
+
raise ValueError("anchor must be finite")
|
| 129 |
+
if not torch.isfinite(target_action[edit_mask.bool()]).all():
|
| 130 |
+
raise ValueError("editable target actions must be finite")
|
| 131 |
+
safe_target = torch.where(edit_mask.bool(), target_action, anchor.detach())
|
| 132 |
+
target_residual = safe_target - anchor.detach()
|
| 133 |
+
articulation_residual = torch.cat(
|
| 134 |
+
(
|
| 135 |
+
target_residual[..., LEFT_ARTICULATION[0] : LEFT_ARTICULATION[1]],
|
| 136 |
+
target_residual[..., RIGHT_ARTICULATION[0] : RIGHT_ARTICULATION[1]],
|
| 137 |
+
),
|
| 138 |
+
dim=-1,
|
| 139 |
+
).masked_fill(~articulation_mask, 0)
|
| 140 |
+
residual_embedding = F.normalize(
|
| 141 |
+
self.residual_projection(articulation_residual.flatten(1)), dim=-1
|
| 142 |
+
)
|
| 143 |
+
|
| 144 |
+
tick_float = decision_seconds.to(dtype=torch.float64) * self.tactile_hz
|
| 145 |
+
ticks = tick_float.round().long()
|
| 146 |
+
if not torch.allclose(
|
| 147 |
+
tick_float, ticks.to(tick_float.dtype), rtol=0, atol=1e-5
|
| 148 |
+
):
|
| 149 |
+
raise ValueError("decision_seconds must lie on the tactile tick grid")
|
| 150 |
+
|
| 151 |
+
total = (touch_embedding.sum() + residual_embedding.sum()) * 0
|
| 152 |
+
eligible_samples = 0
|
| 153 |
+
group_keys = torch.stack((ticks, active_code), dim=1)
|
| 154 |
+
for key in torch.unique(group_keys, dim=0):
|
| 155 |
+
if int(key[1]) == 0:
|
| 156 |
+
continue
|
| 157 |
+
rows = (group_keys == key).all(dim=1).nonzero(as_tuple=True)[0]
|
| 158 |
+
count = int(rows.numel())
|
| 159 |
+
if count < 2:
|
| 160 |
+
continue
|
| 161 |
+
logits = (
|
| 162 |
+
touch_embedding.index_select(0, rows)
|
| 163 |
+
@ residual_embedding.index_select(0, rows).T
|
| 164 |
+
) / self.temperature
|
| 165 |
+
labels = torch.arange(count, device=logits.device)
|
| 166 |
+
group_loss = 0.5 * (
|
| 167 |
+
F.cross_entropy(logits, labels, reduction="sum")
|
| 168 |
+
+ F.cross_entropy(logits.T, labels, reduction="sum")
|
| 169 |
+
)
|
| 170 |
+
total = total + group_loss
|
| 171 |
+
eligible_samples += count
|
| 172 |
+
return total / max(eligible_samples, 1)
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
class CausalHistoryPressureDeltaLoss(nn.Module):
|
| 176 |
+
"""Predict all seven next-frame pressure deltas in an 8-frame history.
|
| 177 |
+
|
| 178 |
+
For transition ``t -> t+1``, the prediction uses only the causal tactile
|
| 179 |
+
history feature at ``t``. Raw pressure at ``t+1`` is a detached supervision
|
| 180 |
+
target and is never passed to the prediction head.
|
| 181 |
+
"""
|
| 182 |
+
|
| 183 |
+
def __init__(
|
| 184 |
+
self,
|
| 185 |
+
*,
|
| 186 |
+
feature_dim: int = 128,
|
| 187 |
+
history_length: int = 8,
|
| 188 |
+
grid_size: int = 21,
|
| 189 |
+
hidden_dim: int = 128,
|
| 190 |
+
target_scale: float = 0.01,
|
| 191 |
+
) -> None:
|
| 192 |
+
super().__init__()
|
| 193 |
+
if feature_dim < 1 or history_length < 2 or grid_size < 1 or hidden_dim < 1:
|
| 194 |
+
raise ValueError(
|
| 195 |
+
"feature dimensions must be positive and history_length >= 2"
|
| 196 |
+
)
|
| 197 |
+
if not math.isfinite(target_scale) or target_scale <= 0:
|
| 198 |
+
raise ValueError("target_scale must be a finite positive train-only scale")
|
| 199 |
+
self.feature_dim = int(feature_dim)
|
| 200 |
+
self.history_length = int(history_length)
|
| 201 |
+
self.grid_size = int(grid_size)
|
| 202 |
+
self.target_scale = float(target_scale)
|
| 203 |
+
self.prediction_head = nn.Sequential(
|
| 204 |
+
nn.LayerNorm(feature_dim),
|
| 205 |
+
nn.Linear(feature_dim, hidden_dim),
|
| 206 |
+
nn.GELU(),
|
| 207 |
+
nn.Linear(hidden_dim, grid_size * grid_size),
|
| 208 |
+
)
|
| 209 |
+
|
| 210 |
+
def forward(
|
| 211 |
+
self,
|
| 212 |
+
history_features: torch.Tensor,
|
| 213 |
+
pressure: torch.Tensor,
|
| 214 |
+
temporal_valid: torch.Tensor,
|
| 215 |
+
edit_mask: torch.Tensor,
|
| 216 |
+
pressure_masks: torch.Tensor,
|
| 217 |
+
) -> torch.Tensor:
|
| 218 |
+
"""Return active-hand, valid-cell masked Smooth L1 over transitions."""
|
| 219 |
+
|
| 220 |
+
batch = len(history_features)
|
| 221 |
+
expected_features = (batch, 2, self.history_length, self.feature_dim)
|
| 222 |
+
expected_pressure = (
|
| 223 |
+
batch,
|
| 224 |
+
self.history_length,
|
| 225 |
+
2,
|
| 226 |
+
self.grid_size,
|
| 227 |
+
self.grid_size,
|
| 228 |
+
)
|
| 229 |
+
if history_features.shape != expected_features:
|
| 230 |
+
raise ValueError(f"history_features must be {expected_features}")
|
| 231 |
+
if pressure.shape != expected_pressure:
|
| 232 |
+
raise ValueError(f"pressure must be {expected_pressure}")
|
| 233 |
+
if temporal_valid.shape != (batch, self.history_length, 2):
|
| 234 |
+
raise ValueError("temporal_valid must be [B,H,2]")
|
| 235 |
+
if pressure_masks.shape != (2, self.grid_size, self.grid_size):
|
| 236 |
+
raise ValueError("pressure_masks must be [2,G,G]")
|
| 237 |
+
if edit_mask.ndim != 3 or len(edit_mask) != batch:
|
| 238 |
+
raise ValueError("edit_mask must be [B,T,A]")
|
| 239 |
+
|
| 240 |
+
# Each output at t is computed only from the causal feature at t. The
|
| 241 |
+
# final feature has no target inside the cached history and is unused.
|
| 242 |
+
prediction = self.prediction_head(history_features[:, :, :-1])
|
| 243 |
+
prediction = prediction.reshape(
|
| 244 |
+
batch, 2, self.history_length - 1, self.grid_size, self.grid_size
|
| 245 |
+
).permute(0, 2, 1, 3, 4)
|
| 246 |
+
|
| 247 |
+
canonical_pressure = torch.nan_to_num(pressure.float()).clone()
|
| 248 |
+
canonical_pressure[:, :, 0] = canonical_pressure[:, :, 0].flip(-1)
|
| 249 |
+
target = (
|
| 250 |
+
canonical_pressure[:, 1:] - canonical_pressure[:, :-1]
|
| 251 |
+
) / self.target_scale
|
| 252 |
+
|
| 253 |
+
active_hands = active_hands_from_edit_mask(edit_mask)
|
| 254 |
+
pair_valid = temporal_valid[:, 1:].bool() & temporal_valid[:, :-1].bool()
|
| 255 |
+
valid_cells = (
|
| 256 |
+
pair_valid[..., None, None]
|
| 257 |
+
& active_hands[:, None, :, None, None]
|
| 258 |
+
& pressure_masks.bool()[None, None].to(device=pressure.device)
|
| 259 |
+
)
|
| 260 |
+
if not bool(valid_cells.any()):
|
| 261 |
+
return prediction.sum() * 0
|
| 262 |
+
safe_target = torch.where(valid_cells, target, prediction.detach())
|
| 263 |
+
loss = F.smooth_l1_loss(prediction, safe_target, reduction="none")
|
| 264 |
+
return loss.masked_fill(~valid_cells, 0).sum() / valid_cells.sum()
|
model/tachin_paired30_cache_20260923/code/src/tactile_editor/config.py
ADDED
|
@@ -0,0 +1,160 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Configuration shared by all tactile-editor variants."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import json
|
| 6 |
+
import math
|
| 7 |
+
import warnings
|
| 8 |
+
from dataclasses import asdict, dataclass, field
|
| 9 |
+
from enum import Enum
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
from typing import Any
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class Variant(str, Enum):
|
| 15 |
+
"""The four controlled integration variants."""
|
| 16 |
+
|
| 17 |
+
V0_BASE = "v0_base"
|
| 18 |
+
V1_POSTHOC = "v1_posthoc_residual"
|
| 19 |
+
V2_EXPERT_TOUCH = "v2_expert_touch"
|
| 20 |
+
V3_MEMORY_TOUCH = "v3_memory_conditioned_touch"
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
class TrainMode(str, Enum):
|
| 24 |
+
"""Parameter policies are independent of the architecture variant."""
|
| 25 |
+
|
| 26 |
+
ADAPTER_ONLY = "adapter_only"
|
| 27 |
+
ENCODER_EDITOR = "encoder_editor"
|
| 28 |
+
EXPERT_TOP = "expert_top"
|
| 29 |
+
ENCODER_TOP = "encoder_top"
|
| 30 |
+
JOINT = "joint_finetune"
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class RuntimeMode(str, Enum):
|
| 34 |
+
"""The runtime scheduler used for action deployment."""
|
| 35 |
+
|
| 36 |
+
PIPELINED = "pipelined"
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
@dataclass(frozen=True)
|
| 40 |
+
class EncoderConfig:
|
| 41 |
+
model_id: str = "MIT-Media-Lab/egotouch-tactile-encoder-p1-v1"
|
| 42 |
+
seed: int = 20260831
|
| 43 |
+
history_frames: int = 8
|
| 44 |
+
grid_size: int = 21
|
| 45 |
+
embedding_dim: int = 64
|
| 46 |
+
canonical_hand: str = "right"
|
| 47 |
+
frozen: bool = True
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
@dataclass(frozen=True)
|
| 51 |
+
class FusionConfig:
|
| 52 |
+
hidden_dim: int = 720
|
| 53 |
+
action_dim: int = 192
|
| 54 |
+
chunk_size: int = 16
|
| 55 |
+
num_heads: int = 8
|
| 56 |
+
injection_layers: tuple[int, ...] = (3, 7, 11, 15)
|
| 57 |
+
posthoc_hidden_dim: int = 256
|
| 58 |
+
posthoc_heads: int = 8
|
| 59 |
+
posthoc_layers: int = 2
|
| 60 |
+
posthoc_dropout: float = 0.1
|
| 61 |
+
posthoc_edit_gate_bias: float = -4.0
|
| 62 |
+
posthoc_touch_gate_init: float = 0.0
|
| 63 |
+
posthoc_use_touch: bool = True
|
| 64 |
+
|
| 65 |
+
def __post_init__(self) -> None:
|
| 66 |
+
if min(self.hidden_dim, self.action_dim, self.chunk_size) <= 0:
|
| 67 |
+
raise ValueError("fusion dimensions must be positive")
|
| 68 |
+
if self.hidden_dim % self.num_heads:
|
| 69 |
+
raise ValueError("hidden_dim must be divisible by num_heads")
|
| 70 |
+
if self.posthoc_hidden_dim % self.posthoc_heads:
|
| 71 |
+
raise ValueError("posthoc_hidden_dim must be divisible by posthoc_heads")
|
| 72 |
+
if not 0.0 <= self.posthoc_dropout < 1.0:
|
| 73 |
+
raise ValueError("posthoc_dropout must be in [0, 1)")
|
| 74 |
+
if not all(
|
| 75 |
+
math.isfinite(x)
|
| 76 |
+
for x in (self.posthoc_edit_gate_bias, self.posthoc_touch_gate_init)
|
| 77 |
+
):
|
| 78 |
+
raise ValueError("gate initialization must be finite")
|
| 79 |
+
if not self.injection_layers:
|
| 80 |
+
raise ValueError("at least one expert injection layer is required")
|
| 81 |
+
if min(self.injection_layers) < 0:
|
| 82 |
+
raise ValueError("injection layer indices must be non-negative")
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
@dataclass(frozen=True)
|
| 86 |
+
class RuntimeConfig:
|
| 87 |
+
mode: RuntimeMode = RuntimeMode.PIPELINED
|
| 88 |
+
control_hz: float = 10.0
|
| 89 |
+
tactile_model_hz: float = 30.0
|
| 90 |
+
action_hz: float = 80.0
|
| 91 |
+
chunk_size: int = 16
|
| 92 |
+
executed_steps: int = 8
|
| 93 |
+
compute_latency_seconds: float = 0.0
|
| 94 |
+
|
| 95 |
+
def __post_init__(self) -> None:
|
| 96 |
+
object.__setattr__(self, "mode", RuntimeMode(self.mode))
|
| 97 |
+
if min(self.control_hz, self.tactile_model_hz, self.action_hz) <= 0:
|
| 98 |
+
raise ValueError("all runtime rates must be positive")
|
| 99 |
+
if not 0 < self.executed_steps <= self.chunk_size:
|
| 100 |
+
raise ValueError("executed_steps must be inside the action chunk")
|
| 101 |
+
if self.compute_latency_seconds < 0:
|
| 102 |
+
raise ValueError("compute_latency_seconds must be non-negative")
|
| 103 |
+
control_period = 1.0 / self.control_hz
|
| 104 |
+
execution_period = self.executed_steps / self.action_hz
|
| 105 |
+
if abs(control_period - execution_period) > 1e-9:
|
| 106 |
+
raise ValueError(
|
| 107 |
+
"executed_steps/action_hz must equal the VLA control period"
|
| 108 |
+
)
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
@dataclass(frozen=True)
|
| 112 |
+
class EditorConfig:
|
| 113 |
+
variant: Variant = Variant.V0_BASE
|
| 114 |
+
train_mode: TrainMode = TrainMode.ADAPTER_ONLY
|
| 115 |
+
encoder: EncoderConfig = field(default_factory=EncoderConfig)
|
| 116 |
+
fusion: FusionConfig = field(default_factory=FusionConfig)
|
| 117 |
+
runtime: RuntimeConfig = field(default_factory=RuntimeConfig)
|
| 118 |
+
|
| 119 |
+
@classmethod
|
| 120 |
+
def from_dict(cls, raw: dict[str, Any]) -> "EditorConfig":
|
| 121 |
+
fusion = dict(raw.get("fusion", {}))
|
| 122 |
+
legacy = {key for key in ("max_residual", "residual_bound") if key in fusion}
|
| 123 |
+
if legacy:
|
| 124 |
+
warnings.warn(
|
| 125 |
+
"Residual amplitude limits were removed. Ignoring legacy fields "
|
| 126 |
+
f"{sorted(legacy)}. Weight shapes remain compatible, but predictions "
|
| 127 |
+
"can differ from the bounded checkpoint. Re-evaluation is required.",
|
| 128 |
+
UserWarning,
|
| 129 |
+
stacklevel=2,
|
| 130 |
+
)
|
| 131 |
+
for key in legacy:
|
| 132 |
+
fusion.pop(key)
|
| 133 |
+
runtime = dict(raw.get("runtime", {}))
|
| 134 |
+
if "vision_hz" in runtime and "control_hz" not in runtime:
|
| 135 |
+
runtime["control_hz"] = runtime.pop("vision_hz")
|
| 136 |
+
return cls(
|
| 137 |
+
variant=Variant(raw.get("variant", Variant.V0_BASE)),
|
| 138 |
+
train_mode=TrainMode(raw.get("train_mode", TrainMode.ADAPTER_ONLY)),
|
| 139 |
+
encoder=EncoderConfig(**raw.get("encoder", {})),
|
| 140 |
+
fusion=FusionConfig(
|
| 141 |
+
**{
|
| 142 |
+
**fusion,
|
| 143 |
+
"injection_layers": tuple(
|
| 144 |
+
fusion.get("injection_layers", (3, 7, 11, 15))
|
| 145 |
+
),
|
| 146 |
+
}
|
| 147 |
+
),
|
| 148 |
+
runtime=RuntimeConfig(**runtime),
|
| 149 |
+
)
|
| 150 |
+
|
| 151 |
+
@classmethod
|
| 152 |
+
def from_json(cls, path: str | Path) -> "EditorConfig":
|
| 153 |
+
return cls.from_dict(json.loads(Path(path).read_text(encoding="utf-8")))
|
| 154 |
+
|
| 155 |
+
def to_dict(self) -> dict[str, Any]:
|
| 156 |
+
result = asdict(self)
|
| 157 |
+
result["variant"] = self.variant.value
|
| 158 |
+
result["train_mode"] = self.train_mode.value
|
| 159 |
+
result["runtime"]["mode"] = self.runtime.mode.value
|
| 160 |
+
return result
|
model/tachin_paired30_cache_20260923/code/src/tactile_editor/contact_filter.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Contact-aware filtering and hand-level action masking for tactile editor data.
|
| 2 |
+
|
| 3 |
+
The selector uses the mean of the five largest masked pressure values for each
|
| 4 |
+
hand. A row is retained when at least one supervised hand is in contact. During
|
| 5 |
+
loading, no-contact hands are masked so the editor cannot change their action.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import bisect
|
| 11 |
+
import hashlib
|
| 12 |
+
import json
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
|
| 15 |
+
import numpy as np
|
| 16 |
+
import torch
|
| 17 |
+
from torch.utils.data import Dataset
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
DEFAULT_CONTACT_THRESHOLD = 0.25
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def contact_gate(
|
| 24 |
+
pressure: torch.Tensor,
|
| 25 |
+
valid: torch.Tensor,
|
| 26 |
+
pressure_masks: torch.Tensor,
|
| 27 |
+
*,
|
| 28 |
+
threshold: float = DEFAULT_CONTACT_THRESHOLD,
|
| 29 |
+
inclusive: bool = True,
|
| 30 |
+
) -> torch.Tensor:
|
| 31 |
+
"""Return ``[batch, 2]`` contact gates from pressure history.
|
| 32 |
+
|
| 33 |
+
``pressure`` is ``[B,T,2,H,W]`` and ``valid`` is ``[B,T,2]``. The gate is
|
| 34 |
+
based on the maximum over valid history frames of the masked top-five mean.
|
| 35 |
+
"""
|
| 36 |
+
values = pressure.float()
|
| 37 |
+
masks = pressure_masks.to(device=values.device, dtype=torch.bool)
|
| 38 |
+
values = values.masked_fill(~masks[None, None], float("-inf"))
|
| 39 |
+
top5 = values.flatten(-2).topk(k=5, dim=-1).values.mean(-1)
|
| 40 |
+
top5 = top5.masked_fill(~valid.to(device=values.device, dtype=torch.bool), float("-inf"))
|
| 41 |
+
score = top5.amax(dim=1)
|
| 42 |
+
return score.ge(float(threshold)) if inclusive else score.gt(float(threshold))
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
class ContactFilteredCache(Dataset):
|
| 46 |
+
"""A view over ``UnifiedWorldCache`` with contact-aware row selection."""
|
| 47 |
+
|
| 48 |
+
def __init__(self, base, pressure_masks: torch.Tensor, indices: np.ndarray, *, threshold: float = DEFAULT_CONTACT_THRESHOLD):
|
| 49 |
+
self.base = base
|
| 50 |
+
self.pressure_masks = pressure_masks.bool().cpu()
|
| 51 |
+
self.selected = np.asarray(indices, dtype=np.int64)
|
| 52 |
+
self.threshold = float(threshold)
|
| 53 |
+
|
| 54 |
+
def __len__(self):
|
| 55 |
+
return len(self.selected)
|
| 56 |
+
|
| 57 |
+
def __getitem__(self, index):
|
| 58 |
+
row = self.base[int(self.selected[index])]
|
| 59 |
+
gate = contact_gate(
|
| 60 |
+
row["touch_pressure"].unsqueeze(0),
|
| 61 |
+
row["touch_valid"].unsqueeze(0),
|
| 62 |
+
self.pressure_masks,
|
| 63 |
+
threshold=self.threshold,
|
| 64 |
+
)[0]
|
| 65 |
+
row["touch_valid"] = row["touch_valid"] & gate.unsqueeze(0)
|
| 66 |
+
if "future_touch_valid" in row:
|
| 67 |
+
row["future_touch_valid"] = row["future_touch_valid"] & gate
|
| 68 |
+
row["action_mask"][:, :69] &= gate[0]
|
| 69 |
+
row["action_mask"][:, 69:138] &= gate[1]
|
| 70 |
+
return row
|
| 71 |
+
|
| 72 |
+
def identity(self):
|
| 73 |
+
identity = self.base.identity() if hasattr(self.base, "identity") else {
|
| 74 |
+
"cache_root": str(getattr(self.base, "cache_root", "unknown")),
|
| 75 |
+
"split": getattr(self.base, "split", "unknown"),
|
| 76 |
+
}
|
| 77 |
+
identity.update(
|
| 78 |
+
contact_threshold=self.threshold,
|
| 79 |
+
selected_rows=len(self),
|
| 80 |
+
selection_sha256=hashlib.sha256(self.selected.tobytes()).hexdigest(),
|
| 81 |
+
)
|
| 82 |
+
return identity
|
| 83 |
+
|
| 84 |
+
def read_column(self, name: str) -> np.ndarray:
|
| 85 |
+
"""Read a column in filtered dataset order, preserving cache APIs."""
|
| 86 |
+
if hasattr(self.base, "read_column"):
|
| 87 |
+
values = self.base.read_column(name)
|
| 88 |
+
else:
|
| 89 |
+
values = np.asarray([self.base[int(i)][name].numpy() for i in self.selected])
|
| 90 |
+
return values
|
| 91 |
+
return np.asarray(values)[self.selected]
|
model/tachin_paired30_cache_20260923/code/src/tactile_editor/direct_residual.py
ADDED
|
@@ -0,0 +1,327 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""End-to-end causal tactile-to-action-residual models.
|
| 2 |
+
|
| 3 |
+
The tactile trunk is trained only by downstream action supervision. It does not
|
| 4 |
+
use a tactile-to-pose retrieval objective. The action branch creates residual
|
| 5 |
+
bases without tactile conditioning; the tactile branch creates coefficients.
|
| 6 |
+
They meet only in the final weighted sum.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import math
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
from torch import nn
|
| 15 |
+
|
| 16 |
+
from .action_layout import (
|
| 17 |
+
ACTION_DIM,
|
| 18 |
+
ARTICULATION_GROUPS,
|
| 19 |
+
FULL_ACTION_GROUPS,
|
| 20 |
+
)
|
| 21 |
+
|
| 22 |
+
DIRECT_INPUT_MODES = ("current", "current_delta", "history_delta", "history")
|
| 23 |
+
COEFFICIENT_SCOPES = ("global", "hand_local")
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def canonical_tactile_channels(
|
| 27 |
+
pressure: torch.Tensor,
|
| 28 |
+
temporal_valid: torch.Tensor,
|
| 29 |
+
pressure_masks: torch.Tensor,
|
| 30 |
+
*,
|
| 31 |
+
include_delta: bool,
|
| 32 |
+
compact: bool = False,
|
| 33 |
+
flip_left: bool = True,
|
| 34 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 35 |
+
"""Return canonical pressure/difference/mask channels and latest validity."""
|
| 36 |
+
|
| 37 |
+
if pressure.ndim != 5 or tuple(pressure.shape[2:]) != (2, 21, 21):
|
| 38 |
+
raise ValueError("pressure must be [B,T,2,21,21]")
|
| 39 |
+
if tuple(temporal_valid.shape) != tuple(pressure.shape[:3]):
|
| 40 |
+
raise ValueError("temporal_valid must be [B,T,2]")
|
| 41 |
+
if tuple(pressure_masks.shape) != (2, 21, 21):
|
| 42 |
+
raise ValueError("pressure_masks must be [2,21,21]")
|
| 43 |
+
|
| 44 |
+
value = torch.nan_to_num(pressure.float()).clone()
|
| 45 |
+
# EgoTouch stores the left sensor in the opposite horizontal convention.
|
| 46 |
+
if flip_left:
|
| 47 |
+
value[:, :, 0] = value[:, :, 0].flip(-1)
|
| 48 |
+
valid = temporal_valid.bool()
|
| 49 |
+
spatial_valid = (
|
| 50 |
+
valid[..., None, None]
|
| 51 |
+
& pressure_masks.bool()[None, None].to(device=value.device)
|
| 52 |
+
)
|
| 53 |
+
value = value * spatial_valid.to(value.dtype)
|
| 54 |
+
difference = torch.zeros_like(value)
|
| 55 |
+
if include_delta and value.shape[1] > 1:
|
| 56 |
+
pair_valid = valid[:, 1:] & valid[:, :-1]
|
| 57 |
+
difference[:, 1:] = (
|
| 58 |
+
value[:, 1:] - value[:, :-1]
|
| 59 |
+
) * pair_valid[..., None, None].to(value.dtype)
|
| 60 |
+
if compact:
|
| 61 |
+
channels = torch.stack((value, spatial_valid.to(value.dtype)), dim=3)
|
| 62 |
+
else:
|
| 63 |
+
channels = torch.stack(
|
| 64 |
+
(value, difference, spatial_valid.to(value.dtype)), dim=3
|
| 65 |
+
)
|
| 66 |
+
return channels, valid[:, -1]
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
class DirectTactileTrunk(nn.Module):
|
| 70 |
+
"""Map raw causal tactile history directly to residual-basis coefficients."""
|
| 71 |
+
|
| 72 |
+
def __init__(
|
| 73 |
+
self,
|
| 74 |
+
pressure_masks: torch.Tensor,
|
| 75 |
+
reference_pressure: torch.Tensor,
|
| 76 |
+
*,
|
| 77 |
+
mode: str,
|
| 78 |
+
coefficient_scope: str = "global",
|
| 79 |
+
residual_bases: int = 8,
|
| 80 |
+
frame_dim: int = 128,
|
| 81 |
+
temporal_hidden: int = 128,
|
| 82 |
+
dropout: float = 0.1,
|
| 83 |
+
flip_left: bool = True,
|
| 84 |
+
) -> None:
|
| 85 |
+
super().__init__()
|
| 86 |
+
if mode not in DIRECT_INPUT_MODES:
|
| 87 |
+
raise ValueError(f"unknown direct tactile input mode: {mode}")
|
| 88 |
+
if coefficient_scope not in COEFFICIENT_SCOPES:
|
| 89 |
+
raise ValueError(f"unknown coefficient scope: {coefficient_scope}")
|
| 90 |
+
if tuple(reference_pressure.shape) != (1, 8, 2, 21, 21):
|
| 91 |
+
raise ValueError("reference_pressure must be [1,8,2,21,21]")
|
| 92 |
+
if residual_bases < 1:
|
| 93 |
+
raise ValueError("residual_bases must be positive")
|
| 94 |
+
self.mode = mode
|
| 95 |
+
self.flip_left = flip_left
|
| 96 |
+
self.coefficient_scope = coefficient_scope
|
| 97 |
+
self.residual_bases = residual_bases
|
| 98 |
+
self.register_buffer("pressure_masks", pressure_masks.bool(), persistent=True)
|
| 99 |
+
self.register_buffer(
|
| 100 |
+
"reference_pressure", reference_pressure.float(), persistent=True
|
| 101 |
+
)
|
| 102 |
+
self.compact_channels = mode == "history"
|
| 103 |
+
input_channels = 2 if self.compact_channels else 3
|
| 104 |
+
self.frame_encoder = nn.Sequential(
|
| 105 |
+
nn.Conv2d(input_channels, 32, 5, padding=2),
|
| 106 |
+
nn.GELU(),
|
| 107 |
+
nn.MaxPool2d(2),
|
| 108 |
+
nn.Conv2d(32, 32, 5, padding=2),
|
| 109 |
+
nn.GELU(),
|
| 110 |
+
nn.MaxPool2d(2),
|
| 111 |
+
nn.Conv2d(32, 32, 5, padding=2),
|
| 112 |
+
nn.GELU(),
|
| 113 |
+
nn.MaxPool2d(2),
|
| 114 |
+
nn.Flatten(),
|
| 115 |
+
nn.Linear(128, frame_dim),
|
| 116 |
+
nn.GELU(),
|
| 117 |
+
)
|
| 118 |
+
self.temporal = (
|
| 119 |
+
nn.GRU(
|
| 120 |
+
frame_dim,
|
| 121 |
+
temporal_hidden,
|
| 122 |
+
num_layers=1,
|
| 123 |
+
batch_first=True,
|
| 124 |
+
)
|
| 125 |
+
if mode in {"history_delta", "history"}
|
| 126 |
+
else None
|
| 127 |
+
)
|
| 128 |
+
output_dim = temporal_hidden if self.temporal is not None else frame_dim
|
| 129 |
+
coefficient_input_dim = (
|
| 130 |
+
output_dim * 2 if coefficient_scope == "global" else output_dim
|
| 131 |
+
)
|
| 132 |
+
self.coefficient_head = nn.Sequential(
|
| 133 |
+
nn.LayerNorm(coefficient_input_dim),
|
| 134 |
+
nn.Dropout(dropout),
|
| 135 |
+
nn.Linear(coefficient_input_dim, residual_bases, bias=False),
|
| 136 |
+
)
|
| 137 |
+
|
| 138 |
+
def _encode(
|
| 139 |
+
self, pressure: torch.Tensor, temporal_valid: torch.Tensor
|
| 140 |
+
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 141 |
+
channels, latest_valid = canonical_tactile_channels(
|
| 142 |
+
pressure,
|
| 143 |
+
temporal_valid,
|
| 144 |
+
self.pressure_masks,
|
| 145 |
+
include_delta=self.mode in {"current_delta", "history_delta"},
|
| 146 |
+
compact=self.compact_channels,
|
| 147 |
+
flip_left=self.flip_left,
|
| 148 |
+
)
|
| 149 |
+
batch, history, hands = channels.shape[:3]
|
| 150 |
+
frames = self.frame_encoder(
|
| 151 |
+
channels.reshape(batch * history * hands, channels.shape[3], 21, 21)
|
| 152 |
+
)
|
| 153 |
+
frames = frames.reshape(batch, history, hands, -1).permute(0, 2, 1, 3)
|
| 154 |
+
frame_valid = temporal_valid.bool().permute(0, 2, 1)[..., None]
|
| 155 |
+
frames = frames * frame_valid.to(frames.dtype)
|
| 156 |
+
if self.temporal is None:
|
| 157 |
+
history_features = frames
|
| 158 |
+
encoded = history_features[:, :, -1]
|
| 159 |
+
else:
|
| 160 |
+
sequence = frames.reshape(batch * hands, history, -1)
|
| 161 |
+
self.temporal.flatten_parameters()
|
| 162 |
+
history_features = self.temporal(sequence)[0]
|
| 163 |
+
history_features = history_features.reshape(batch, hands, history, -1)
|
| 164 |
+
encoded = history_features[:, :, -1]
|
| 165 |
+
return encoded, latest_valid, history_features
|
| 166 |
+
|
| 167 |
+
def forward(
|
| 168 |
+
self, pressure: torch.Tensor, temporal_valid: torch.Tensor
|
| 169 |
+
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 170 |
+
batch = len(pressure)
|
| 171 |
+
reference_pressure = self.reference_pressure.to(device=pressure.device).expand(
|
| 172 |
+
batch, -1, -1, -1, -1
|
| 173 |
+
)
|
| 174 |
+
reference_valid = torch.ones(
|
| 175 |
+
batch,
|
| 176 |
+
self.reference_pressure.shape[1],
|
| 177 |
+
2,
|
| 178 |
+
dtype=torch.bool,
|
| 179 |
+
device=pressure.device,
|
| 180 |
+
)
|
| 181 |
+
# Encode matched and reference inputs in one call. Identical tensors then
|
| 182 |
+
# produce bitwise-identical features even when convolution algorithms are
|
| 183 |
+
# sensitive to batch shape.
|
| 184 |
+
combined_pressure = torch.cat((pressure, reference_pressure), dim=0)
|
| 185 |
+
combined_valid = torch.cat((temporal_valid, reference_valid), dim=0)
|
| 186 |
+
combined, combined_hand_valid, combined_history = self._encode(
|
| 187 |
+
combined_pressure, combined_valid
|
| 188 |
+
)
|
| 189 |
+
encoded, reference = combined.split(batch, dim=0)
|
| 190 |
+
encoded_history, reference_history = combined_history.split(batch, dim=0)
|
| 191 |
+
hand_valid = combined_hand_valid[:batch]
|
| 192 |
+
centered = (encoded - reference).mul(hand_valid[..., None].to(encoded.dtype))
|
| 193 |
+
history_valid = temporal_valid.bool().permute(0, 2, 1)[..., None]
|
| 194 |
+
centered_history = (encoded_history - reference_history).mul(
|
| 195 |
+
history_valid.to(encoded_history.dtype)
|
| 196 |
+
)
|
| 197 |
+
if self.coefficient_scope == "global":
|
| 198 |
+
coefficients = self.coefficient_head(centered.flatten(1))
|
| 199 |
+
active = hand_valid.any(1, keepdim=True) & (
|
| 200 |
+
centered.abs().sum((1, 2), keepdim=False)[:, None] > 0
|
| 201 |
+
)
|
| 202 |
+
coefficients = coefficients * active.to(coefficients.dtype)
|
| 203 |
+
else:
|
| 204 |
+
coefficients = self.coefficient_head(centered)
|
| 205 |
+
active = hand_valid & (centered.abs().sum(-1) > 0)
|
| 206 |
+
coefficients = coefficients * active[..., None].to(coefficients.dtype)
|
| 207 |
+
return coefficients, hand_valid, centered, centered_history
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
class ActionResidualBasisTrunk(nn.Module):
|
| 211 |
+
"""Create action-conditioned bases without access to tactile inputs."""
|
| 212 |
+
|
| 213 |
+
def __init__(
|
| 214 |
+
self,
|
| 215 |
+
action_mean: torch.Tensor,
|
| 216 |
+
action_std: torch.Tensor,
|
| 217 |
+
*,
|
| 218 |
+
residual_bases: int = 8,
|
| 219 |
+
hidden: int = 128,
|
| 220 |
+
dropout: float = 0.1,
|
| 221 |
+
) -> None:
|
| 222 |
+
super().__init__()
|
| 223 |
+
if tuple(action_mean.shape) != (192,) or tuple(action_std.shape) != (192,):
|
| 224 |
+
raise ValueError("action statistics must have shape [192]")
|
| 225 |
+
self.residual_bases = residual_bases
|
| 226 |
+
self.register_buffer("action_mean", action_mean.float(), persistent=True)
|
| 227 |
+
self.register_buffer(
|
| 228 |
+
"action_std", action_std.float().clamp_min(1e-3), persistent=True
|
| 229 |
+
)
|
| 230 |
+
self.action_projection = nn.Linear(192, hidden)
|
| 231 |
+
self.position = nn.Parameter(torch.randn(1, 16, hidden) * 0.02)
|
| 232 |
+
self.time_projection = nn.Linear(1, hidden)
|
| 233 |
+
layer = nn.TransformerEncoderLayer(
|
| 234 |
+
hidden,
|
| 235 |
+
nhead=4,
|
| 236 |
+
dim_feedforward=hidden * 4,
|
| 237 |
+
dropout=dropout,
|
| 238 |
+
activation="gelu",
|
| 239 |
+
batch_first=True,
|
| 240 |
+
norm_first=True,
|
| 241 |
+
)
|
| 242 |
+
self.trunk = nn.TransformerEncoder(
|
| 243 |
+
layer, num_layers=1, enable_nested_tensor=False
|
| 244 |
+
)
|
| 245 |
+
self.basis_head = nn.Linear(hidden, residual_bases * 192)
|
| 246 |
+
nn.init.zeros_(self.basis_head.weight)
|
| 247 |
+
nn.init.zeros_(self.basis_head.bias)
|
| 248 |
+
|
| 249 |
+
def forward(self, anchor: torch.Tensor, elapsed: torch.Tensor) -> torch.Tensor:
|
| 250 |
+
if tuple(anchor.shape[1:]) != (16, 192):
|
| 251 |
+
raise ValueError("anchor must be [B,16,192]")
|
| 252 |
+
hidden = self.action_projection(
|
| 253 |
+
(anchor.detach() - self.action_mean) / self.action_std
|
| 254 |
+
)
|
| 255 |
+
hidden = hidden + self.position + self.time_projection(
|
| 256 |
+
elapsed.float()[:, None, None] * 10
|
| 257 |
+
)
|
| 258 |
+
hidden = self.trunk(hidden)
|
| 259 |
+
bases = self.basis_head(hidden).reshape(
|
| 260 |
+
len(anchor), 16, self.residual_bases, 192
|
| 261 |
+
)
|
| 262 |
+
return bases.permute(0, 2, 1, 3)
|
| 263 |
+
|
| 264 |
+
|
| 265 |
+
class DirectTactileResidualEditor(nn.Module):
|
| 266 |
+
"""Predict a tactile-only low-rank correction to a frozen action anchor."""
|
| 267 |
+
|
| 268 |
+
def __init__(
|
| 269 |
+
self,
|
| 270 |
+
action_mean: torch.Tensor,
|
| 271 |
+
action_std: torch.Tensor,
|
| 272 |
+
pressure_masks: torch.Tensor,
|
| 273 |
+
reference_pressure: torch.Tensor,
|
| 274 |
+
*,
|
| 275 |
+
mode: str,
|
| 276 |
+
residual_bases: int = 8,
|
| 277 |
+
articulation_only: bool = True,
|
| 278 |
+
flip_left: bool = True,
|
| 279 |
+
) -> None:
|
| 280 |
+
super().__init__()
|
| 281 |
+
self.tactile = DirectTactileTrunk(
|
| 282 |
+
pressure_masks,
|
| 283 |
+
reference_pressure,
|
| 284 |
+
mode=mode,
|
| 285 |
+
residual_bases=residual_bases,
|
| 286 |
+
flip_left=flip_left,
|
| 287 |
+
)
|
| 288 |
+
self.action = ActionResidualBasisTrunk(
|
| 289 |
+
action_mean, action_std, residual_bases=residual_bases
|
| 290 |
+
)
|
| 291 |
+
component_mask = torch.zeros(1, 1, ACTION_DIM, dtype=torch.bool)
|
| 292 |
+
groups = ARTICULATION_GROUPS if articulation_only else FULL_ACTION_GROUPS
|
| 293 |
+
for start, stop in groups:
|
| 294 |
+
component_mask[..., start:stop] = True
|
| 295 |
+
self.register_buffer("component_mask", component_mask, persistent=True)
|
| 296 |
+
self.residual_bases = residual_bases
|
| 297 |
+
|
| 298 |
+
def forward(
|
| 299 |
+
self,
|
| 300 |
+
anchor: torch.Tensor,
|
| 301 |
+
edit_mask: torch.Tensor,
|
| 302 |
+
pressure: torch.Tensor,
|
| 303 |
+
temporal_valid: torch.Tensor,
|
| 304 |
+
elapsed: torch.Tensor,
|
| 305 |
+
) -> tuple[torch.Tensor, dict[str, torch.Tensor]]:
|
| 306 |
+
coefficients, hand_valid, tactile_features, tactile_history_features = self.tactile(
|
| 307 |
+
pressure, temporal_valid
|
| 308 |
+
)
|
| 309 |
+
bases = self.action(anchor, elapsed)
|
| 310 |
+
raw_delta = (
|
| 311 |
+
coefficients[:, :, None, None] * bases
|
| 312 |
+
).sum(1) / math.sqrt(self.residual_bases)
|
| 313 |
+
allowed = edit_mask.bool() & self.component_mask
|
| 314 |
+
# The action layout is [left 69 | right 69 | padding].
|
| 315 |
+
allowed = allowed.clone()
|
| 316 |
+
allowed[..., :69] &= hand_valid[:, 0, None, None]
|
| 317 |
+
allowed[..., 69:138] &= hand_valid[:, 1, None, None]
|
| 318 |
+
delta = raw_delta * allowed.to(raw_delta.dtype)
|
| 319 |
+
return anchor.detach() + delta, {
|
| 320 |
+
"effective_delta": delta,
|
| 321 |
+
"coefficients": coefficients,
|
| 322 |
+
"bases": bases,
|
| 323 |
+
"allowed_mask": allowed,
|
| 324 |
+
"hand_valid": hand_valid,
|
| 325 |
+
"tactile_features": tactile_features,
|
| 326 |
+
"tactile_history_features": tactile_history_features,
|
| 327 |
+
}
|
model/tachin_paired30_cache_20260923/code/src/tactile_editor/editor_backends.py
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Unified tactile-editor router and backend adapters."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from typing import Any
|
| 6 |
+
from dataclasses import replace
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
from torch import nn
|
| 10 |
+
|
| 11 |
+
from .editor_interface import BaseActionContext, EditorOutput
|
| 12 |
+
from .direct_residual import DirectTactileResidualEditor
|
| 13 |
+
from .unified_world import UnifiedWorldTactileEditor
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class MemoryFreeTactileEditor(nn.Module):
|
| 17 |
+
"""Action-conditioned tactile residual editor for models without memory."""
|
| 18 |
+
|
| 19 |
+
requires_memory = False
|
| 20 |
+
|
| 21 |
+
def __init__(self, editor: DirectTactileResidualEditor) -> None:
|
| 22 |
+
super().__init__()
|
| 23 |
+
self.editor = editor
|
| 24 |
+
|
| 25 |
+
def edit(
|
| 26 |
+
self,
|
| 27 |
+
context: BaseActionContext,
|
| 28 |
+
pressure: torch.Tensor,
|
| 29 |
+
temporal_valid: torch.Tensor,
|
| 30 |
+
decision_seconds: torch.Tensor,
|
| 31 |
+
**_: Any,
|
| 32 |
+
) -> EditorOutput:
|
| 33 |
+
if context.has_memory:
|
| 34 |
+
raise ValueError("MemoryFreeTactileEditor cannot consume memory_tokens")
|
| 35 |
+
final_action, details = self.editor(
|
| 36 |
+
context.base_action,
|
| 37 |
+
context.effective_edit_mask,
|
| 38 |
+
pressure,
|
| 39 |
+
temporal_valid,
|
| 40 |
+
decision_seconds,
|
| 41 |
+
)
|
| 42 |
+
return EditorOutput(
|
| 43 |
+
final_action=final_action,
|
| 44 |
+
delta_action=details["effective_delta"],
|
| 45 |
+
allowed_mask=details["allowed_mask"],
|
| 46 |
+
backend="memory_free",
|
| 47 |
+
auxiliary_outputs=details,
|
| 48 |
+
)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
class NativeFinalActionEditor(nn.Module):
|
| 52 |
+
"""Pass through a model that already performed its native tactile update.
|
| 53 |
+
|
| 54 |
+
T-Rex (and eventually a native T-WLA predictor) returns a tactile-refined
|
| 55 |
+
action, rather than a vision-only anchor plus reusable memory tokens. The
|
| 56 |
+
pass-through keeps the common ``EditorOutput`` contract without applying a
|
| 57 |
+
second tactile residual editor.
|
| 58 |
+
"""
|
| 59 |
+
|
| 60 |
+
requires_memory = False
|
| 61 |
+
|
| 62 |
+
def edit(
|
| 63 |
+
self,
|
| 64 |
+
context: BaseActionContext,
|
| 65 |
+
pressure: torch.Tensor,
|
| 66 |
+
temporal_valid: torch.Tensor,
|
| 67 |
+
decision_seconds: torch.Tensor,
|
| 68 |
+
**_: Any,
|
| 69 |
+
) -> EditorOutput:
|
| 70 |
+
del pressure, temporal_valid, decision_seconds
|
| 71 |
+
return EditorOutput(
|
| 72 |
+
final_action=context.base_action,
|
| 73 |
+
delta_action=torch.zeros_like(context.base_action),
|
| 74 |
+
allowed_mask=context.effective_edit_mask,
|
| 75 |
+
backend="native_tactile",
|
| 76 |
+
auxiliary_outputs={"native_tactile_policy": True},
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
class MemoryW0W3Editor(nn.Module):
|
| 81 |
+
"""Adapter exposing the existing memory-conditioned W0--W3 editor."""
|
| 82 |
+
|
| 83 |
+
requires_memory = True
|
| 84 |
+
|
| 85 |
+
def __init__(self, editor: UnifiedWorldTactileEditor) -> None:
|
| 86 |
+
super().__init__()
|
| 87 |
+
self.editor = editor
|
| 88 |
+
|
| 89 |
+
def edit(
|
| 90 |
+
self,
|
| 91 |
+
context: BaseActionContext,
|
| 92 |
+
pressure: torch.Tensor,
|
| 93 |
+
temporal_valid: torch.Tensor,
|
| 94 |
+
decision_seconds: torch.Tensor,
|
| 95 |
+
**kwargs: Any,
|
| 96 |
+
) -> EditorOutput:
|
| 97 |
+
if not context.has_memory:
|
| 98 |
+
raise ValueError("MemoryW0W3Editor requires memory_tokens")
|
| 99 |
+
# Runtime action editing cannot consume future supervision. Training
|
| 100 |
+
# callers can explicitly enable auxiliary world predictions.
|
| 101 |
+
kwargs.setdefault("compute_auxiliary", False)
|
| 102 |
+
output = self.editor(
|
| 103 |
+
base_action=context.base_action,
|
| 104 |
+
edit_mask=context.effective_edit_mask,
|
| 105 |
+
pressure=pressure,
|
| 106 |
+
temporal_valid=temporal_valid,
|
| 107 |
+
decision_seconds=decision_seconds,
|
| 108 |
+
final_memory=context.memory_tokens,
|
| 109 |
+
image_tokens=context.image_tokens,
|
| 110 |
+
**kwargs,
|
| 111 |
+
)
|
| 112 |
+
auxiliary = {
|
| 113 |
+
"tactile_features": output.tactile_features,
|
| 114 |
+
"hand_valid": output.hand_valid,
|
| 115 |
+
"tactile_velocity": output.tactile_velocity,
|
| 116 |
+
"tactile_target": output.tactile_target,
|
| 117 |
+
"tactile_valid": output.tactile_valid,
|
| 118 |
+
"rgb": output.rgb,
|
| 119 |
+
}
|
| 120 |
+
return EditorOutput(
|
| 121 |
+
final_action=output.final_action,
|
| 122 |
+
delta_action=output.delta_action,
|
| 123 |
+
allowed_mask=output.allowed_mask,
|
| 124 |
+
backend=f"memory_{self.editor.world_mode.value}",
|
| 125 |
+
auxiliary_outputs=auxiliary,
|
| 126 |
+
)
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
class TactileEditorRouter(nn.Module):
|
| 130 |
+
"""Select an explicit shared/memory experiment, or preserve legacy auto routing."""
|
| 131 |
+
|
| 132 |
+
def __init__(
|
| 133 |
+
self,
|
| 134 |
+
memory_backend: MemoryW0W3Editor | None,
|
| 135 |
+
memory_free_backend: MemoryFreeTactileEditor | None,
|
| 136 |
+
native_final_backend: NativeFinalActionEditor | None = None,
|
| 137 |
+
*,
|
| 138 |
+
route: str = "auto",
|
| 139 |
+
) -> None:
|
| 140 |
+
super().__init__()
|
| 141 |
+
if memory_backend is None and memory_free_backend is None:
|
| 142 |
+
raise ValueError("at least one tactile-editor backend is required")
|
| 143 |
+
self.memory_backend = memory_backend
|
| 144 |
+
self.memory_free_backend = memory_free_backend
|
| 145 |
+
self.native_final_backend = native_final_backend or NativeFinalActionEditor()
|
| 146 |
+
if route not in ("auto", "shared", "memory_ablation"):
|
| 147 |
+
raise ValueError("route must be auto, shared, or memory_ablation")
|
| 148 |
+
self.route = route
|
| 149 |
+
|
| 150 |
+
def edit(
|
| 151 |
+
self,
|
| 152 |
+
context: BaseActionContext,
|
| 153 |
+
pressure: torch.Tensor,
|
| 154 |
+
temporal_valid: torch.Tensor,
|
| 155 |
+
decision_seconds: torch.Tensor,
|
| 156 |
+
**kwargs: Any,
|
| 157 |
+
) -> EditorOutput:
|
| 158 |
+
if self.route == "shared":
|
| 159 |
+
if self.memory_free_backend is None:
|
| 160 |
+
raise ValueError("shared route requires a memory-free backend")
|
| 161 |
+
# The base retains its native memory. Only the editor loses access.
|
| 162 |
+
context = replace(context, memory_tokens=None, image_tokens=None)
|
| 163 |
+
return self.memory_free_backend.edit(
|
| 164 |
+
context, pressure, temporal_valid, decision_seconds, **kwargs
|
| 165 |
+
)
|
| 166 |
+
if self.route == "memory_ablation":
|
| 167 |
+
if context.model_id != "twla" or not context.has_memory or self.memory_backend is None:
|
| 168 |
+
raise ValueError("memory_ablation requires twla, memory tokens, and a memory backend")
|
| 169 |
+
return self.memory_backend.edit(
|
| 170 |
+
context, pressure, temporal_valid, decision_seconds, **kwargs
|
| 171 |
+
)
|
| 172 |
+
if context.metadata.get("action_is_final", False):
|
| 173 |
+
return self.native_final_backend.edit(
|
| 174 |
+
context, pressure, temporal_valid, decision_seconds, **kwargs
|
| 175 |
+
)
|
| 176 |
+
if context.has_memory:
|
| 177 |
+
if self.memory_backend is None:
|
| 178 |
+
raise ValueError("memory_tokens were supplied but no memory backend is configured")
|
| 179 |
+
return self.memory_backend.edit(
|
| 180 |
+
context, pressure, temporal_valid, decision_seconds, **kwargs
|
| 181 |
+
)
|
| 182 |
+
if self.memory_free_backend is None:
|
| 183 |
+
raise ValueError("no memory-free backend is configured for this context")
|
| 184 |
+
return self.memory_free_backend.edit(
|
| 185 |
+
context, pressure, temporal_valid, decision_seconds, **kwargs
|
| 186 |
+
)
|
| 187 |
+
|
| 188 |
+
def forward(
|
| 189 |
+
self,
|
| 190 |
+
context: BaseActionContext,
|
| 191 |
+
pressure: torch.Tensor,
|
| 192 |
+
temporal_valid: torch.Tensor,
|
| 193 |
+
decision_seconds: torch.Tensor,
|
| 194 |
+
**kwargs: Any,
|
| 195 |
+
) -> EditorOutput:
|
| 196 |
+
return self.edit(context, pressure, temporal_valid, decision_seconds, **kwargs)
|
model/tachin_paired30_cache_20260923/code/src/tactile_editor/editor_interface.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Shared model-to-tactile-editor interface.
|
| 2 |
+
|
| 3 |
+
VLA implementations differ in their internal memory representation. They all
|
| 4 |
+
expose the same normalized action context here, while editor backends decide
|
| 5 |
+
whether to use optional memory/image tokens.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
from dataclasses import dataclass, field
|
| 11 |
+
from typing import Any, Protocol
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
|
| 15 |
+
from .action_layout import ACTION_DIM, CHUNK_SIZE, DEFAULT_ACTION_LAYOUT
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
@dataclass(frozen=True)
|
| 19 |
+
class BaseActionContext:
|
| 20 |
+
"""Frozen output of a model-specific VLA adapter."""
|
| 21 |
+
|
| 22 |
+
base_action: torch.Tensor
|
| 23 |
+
action_mask: torch.Tensor
|
| 24 |
+
model_id: str
|
| 25 |
+
normalization_id: str
|
| 26 |
+
memory_tokens: torch.Tensor | None = None
|
| 27 |
+
image_tokens: torch.Tensor | None = None
|
| 28 |
+
edit_mask: torch.Tensor | None = None
|
| 29 |
+
metadata: dict[str, Any] = field(default_factory=dict)
|
| 30 |
+
|
| 31 |
+
def __post_init__(self) -> None:
|
| 32 |
+
DEFAULT_ACTION_LAYOUT.validate(self.base_action, name="base_action")
|
| 33 |
+
if self.action_mask.shape != self.base_action.shape:
|
| 34 |
+
raise ValueError("action_mask must match base_action")
|
| 35 |
+
if self.edit_mask is not None and self.edit_mask.shape != self.base_action.shape:
|
| 36 |
+
raise ValueError("edit_mask must match base_action")
|
| 37 |
+
if self.action_mask[..., 138:].any():
|
| 38 |
+
raise ValueError("action_mask cannot mark padding dimensions valid")
|
| 39 |
+
if self.edit_mask is not None and self.edit_mask[..., 138:].any():
|
| 40 |
+
raise ValueError("edit_mask cannot mark padding dimensions editable")
|
| 41 |
+
if not self.base_action.is_floating_point() or not torch.isfinite(self.base_action).all():
|
| 42 |
+
raise ValueError("base_action must be finite floating-point values")
|
| 43 |
+
if self.memory_tokens is not None:
|
| 44 |
+
if self.memory_tokens.ndim != 3 or self.memory_tokens.shape[0] != self.base_action.shape[0]:
|
| 45 |
+
raise ValueError("memory_tokens must be [B,N,D]")
|
| 46 |
+
if self.image_tokens is not None and self.image_tokens.shape[0] != self.base_action.shape[0]:
|
| 47 |
+
raise ValueError("image_tokens batch size must match base_action")
|
| 48 |
+
if not self.model_id or not self.normalization_id:
|
| 49 |
+
raise ValueError("model_id and normalization_id are required")
|
| 50 |
+
|
| 51 |
+
@property
|
| 52 |
+
def effective_edit_mask(self) -> torch.Tensor:
|
| 53 |
+
mask = self.action_mask.bool()
|
| 54 |
+
if self.edit_mask is not None:
|
| 55 |
+
mask = mask & self.edit_mask.bool()
|
| 56 |
+
return mask
|
| 57 |
+
|
| 58 |
+
@property
|
| 59 |
+
def has_memory(self) -> bool:
|
| 60 |
+
return self.memory_tokens is not None
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
@dataclass(frozen=True)
|
| 64 |
+
class EditorOutput:
|
| 65 |
+
"""Common output of every tactile editor backend."""
|
| 66 |
+
|
| 67 |
+
final_action: torch.Tensor
|
| 68 |
+
delta_action: torch.Tensor
|
| 69 |
+
allowed_mask: torch.Tensor
|
| 70 |
+
backend: str
|
| 71 |
+
auxiliary_outputs: dict[str, Any] = field(default_factory=dict)
|
| 72 |
+
|
| 73 |
+
def __post_init__(self) -> None:
|
| 74 |
+
if self.final_action.shape != self.delta_action.shape:
|
| 75 |
+
raise ValueError("final_action and delta_action must match")
|
| 76 |
+
if self.allowed_mask.shape != self.final_action.shape:
|
| 77 |
+
raise ValueError("allowed_mask must match final_action")
|
| 78 |
+
DEFAULT_ACTION_LAYOUT.validate(self.final_action, name="final_action")
|
| 79 |
+
if not self.backend:
|
| 80 |
+
raise ValueError("backend name is required")
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
class TactileEditorBackend(Protocol):
|
| 84 |
+
"""Structural protocol implemented by memory and memory-free backends."""
|
| 85 |
+
|
| 86 |
+
requires_memory: bool
|
| 87 |
+
|
| 88 |
+
def edit(
|
| 89 |
+
self,
|
| 90 |
+
context: BaseActionContext,
|
| 91 |
+
pressure: torch.Tensor,
|
| 92 |
+
temporal_valid: torch.Tensor,
|
| 93 |
+
decision_seconds: torch.Tensor,
|
| 94 |
+
**kwargs: Any,
|
| 95 |
+
) -> EditorOutput: ...
|
| 96 |
+
|
model/tachin_paired30_cache_20260923/code/src/tactile_editor/encoder.py
ADDED
|
@@ -0,0 +1,209 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The stable bridge from two EgoTouch glove streams to two tactile tokens."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from dataclasses import dataclass
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
from torch import nn
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
@dataclass
|
| 13 |
+
class TouchFeatures:
|
| 14 |
+
tokens: torch.Tensor # [B, 2, 64], left then right
|
| 15 |
+
valid: torch.Tensor # [B, 2]
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def _selected_encoder_api():
|
| 19 |
+
try:
|
| 20 |
+
from selected_encoder import ( # type: ignore
|
| 21 |
+
DEFAULT_MODEL_ID,
|
| 22 |
+
DEFAULT_SEED,
|
| 23 |
+
SelectedTactileEncoder,
|
| 24 |
+
canonicalize_raw_touch,
|
| 25 |
+
)
|
| 26 |
+
except ImportError as exc: # pragma: no cover - exercised by packaging smoke
|
| 27 |
+
raise ImportError(
|
| 28 |
+
"Selected EgoTouch encoder is unavailable. Add the repository's Tactile "
|
| 29 |
+
"directory to PYTHONPATH before importing tactile_editor."
|
| 30 |
+
) from exc
|
| 31 |
+
return (
|
| 32 |
+
DEFAULT_MODEL_ID,
|
| 33 |
+
DEFAULT_SEED,
|
| 34 |
+
SelectedTactileEncoder,
|
| 35 |
+
canonicalize_raw_touch,
|
| 36 |
+
)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def load_huggingface_encoder(
|
| 40 |
+
*,
|
| 41 |
+
model_id: str = "MIT-Media-Lab/egotouch-tactile-encoder-p1-v1",
|
| 42 |
+
seed: int = 20260831,
|
| 43 |
+
cache_dir: str | Path | None = None,
|
| 44 |
+
device: str | torch.device = "cpu",
|
| 45 |
+
history_frames: int = 8,
|
| 46 |
+
) -> nn.Module:
|
| 47 |
+
"""Download only the standalone tactile weights and masks, then load strictly."""
|
| 48 |
+
|
| 49 |
+
try:
|
| 50 |
+
from huggingface_hub import snapshot_download
|
| 51 |
+
except ImportError as exc: # pragma: no cover - optional network path
|
| 52 |
+
raise ImportError(
|
| 53 |
+
"huggingface_hub is required for remote encoder loading"
|
| 54 |
+
) from exc
|
| 55 |
+
_, _, selected_cls, _ = _selected_encoder_api()
|
| 56 |
+
if cache_dir is not None:
|
| 57 |
+
Path(cache_dir).mkdir(parents=True, exist_ok=True)
|
| 58 |
+
root = snapshot_download(
|
| 59 |
+
repo_id=model_id,
|
| 60 |
+
cache_dir=str(cache_dir) if cache_dir is not None else None,
|
| 61 |
+
allow_patterns=[
|
| 62 |
+
"pressure_masks.npy",
|
| 63 |
+
f"seed-{seed}/tactile_encoder.safetensors",
|
| 64 |
+
],
|
| 65 |
+
)
|
| 66 |
+
return selected_cls.from_pretrained_dir(
|
| 67 |
+
root, seed=seed, device=device, history_frames=history_frames
|
| 68 |
+
)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def load_formal_t2p_encoder(
|
| 72 |
+
checkpoint: str | Path,
|
| 73 |
+
*,
|
| 74 |
+
history_frames: int,
|
| 75 |
+
device: str | torch.device = "cpu",
|
| 76 |
+
) -> nn.Module:
|
| 77 |
+
"""Load only the tactile tower from a formal P1 T2P checkpoint.
|
| 78 |
+
|
| 79 |
+
The pose tower and contrastive temperature are training-only components.
|
| 80 |
+
The returned module has the same normalized 64-D runtime contract as the
|
| 81 |
+
published selected encoder.
|
| 82 |
+
"""
|
| 83 |
+
|
| 84 |
+
_, _, selected_cls, _ = _selected_encoder_api()
|
| 85 |
+
from egotouch_formal.model import TactileEncoder
|
| 86 |
+
|
| 87 |
+
payload = torch.load(checkpoint, map_location="cpu", weights_only=False)
|
| 88 |
+
config = payload.get("config", {})
|
| 89 |
+
if config.get("task") != "p1":
|
| 90 |
+
raise ValueError(f"formal checkpoint must be P1, got {config.get('task')!r}")
|
| 91 |
+
trained_touch_history = int(config.get("tactile_history_frames", 8))
|
| 92 |
+
if trained_touch_history != int(history_frames):
|
| 93 |
+
raise ValueError(
|
| 94 |
+
f"checkpoint history is {trained_touch_history}, requested {history_frames}"
|
| 95 |
+
)
|
| 96 |
+
source = payload.get("model")
|
| 97 |
+
if not isinstance(source, dict):
|
| 98 |
+
raise ValueError("formal checkpoint has no model state")
|
| 99 |
+
tactile_state = {
|
| 100 |
+
key.removeprefix("tactile."): value
|
| 101 |
+
for key, value in source.items()
|
| 102 |
+
if key.startswith("tactile.")
|
| 103 |
+
}
|
| 104 |
+
encoder = TactileEncoder(embed_dim=int(config.get("embed_dim", 64)))
|
| 105 |
+
encoder.load_state_dict(tactile_state, strict=True)
|
| 106 |
+
pressure_masks = source.get("pressure_masks")
|
| 107 |
+
if pressure_masks is None:
|
| 108 |
+
raise ValueError("formal checkpoint does not contain pressure_masks")
|
| 109 |
+
return selected_cls(
|
| 110 |
+
encoder,
|
| 111 |
+
pressure_masks,
|
| 112 |
+
history_frames=history_frames,
|
| 113 |
+
).to(device)
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def initialize_random_t2p_encoder(
|
| 117 |
+
pressure_masks: torch.Tensor,
|
| 118 |
+
*,
|
| 119 |
+
history_frames: int,
|
| 120 |
+
device: str | torch.device = "cpu",
|
| 121 |
+
) -> nn.Module:
|
| 122 |
+
"""Construct the P1 tactile architecture without pretrained weights."""
|
| 123 |
+
|
| 124 |
+
_, _, selected_cls, _ = _selected_encoder_api()
|
| 125 |
+
from egotouch_formal.model import TactileEncoder
|
| 126 |
+
|
| 127 |
+
return selected_cls(
|
| 128 |
+
TactileEncoder(embed_dim=64),
|
| 129 |
+
pressure_masks,
|
| 130 |
+
history_frames=history_frames,
|
| 131 |
+
).to(device)
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
class DualHandTactileEncoder(nn.Module):
|
| 135 |
+
"""Run one shared encoder over left/right causal histories.
|
| 136 |
+
|
| 137 |
+
Raw input keeps the online cache contract ``[B,8,2,21,21]``. The wrapped
|
| 138 |
+
checkpoint sees the latest ``history_frames`` as ``[B*2,T,21,21]`` plus a
|
| 139 |
+
side id. Left pressure is flipped exactly once when ``canonicalized=False``.
|
| 140 |
+
"""
|
| 141 |
+
|
| 142 |
+
def __init__(self, encoder: nn.Module, *, freeze: bool = True) -> None:
|
| 143 |
+
super().__init__()
|
| 144 |
+
self.encoder = encoder
|
| 145 |
+
self.history_frames = int(getattr(encoder, "history_frames", 8))
|
| 146 |
+
if not 1 <= self.history_frames <= 8:
|
| 147 |
+
raise ValueError("editor bridge supports history_frames in [1,8]")
|
| 148 |
+
self.set_frozen(freeze)
|
| 149 |
+
|
| 150 |
+
def set_frozen(self, frozen: bool) -> None:
|
| 151 |
+
for parameter in self.encoder.parameters():
|
| 152 |
+
parameter.requires_grad = not frozen
|
| 153 |
+
if frozen:
|
| 154 |
+
self.encoder.eval()
|
| 155 |
+
|
| 156 |
+
def train(self, mode: bool = True):
|
| 157 |
+
super().train(mode)
|
| 158 |
+
if not any(parameter.requires_grad for parameter in self.encoder.parameters()):
|
| 159 |
+
self.encoder.eval()
|
| 160 |
+
return self
|
| 161 |
+
|
| 162 |
+
def forward(
|
| 163 |
+
self,
|
| 164 |
+
pressure: torch.Tensor,
|
| 165 |
+
temporal_valid: torch.Tensor | None = None,
|
| 166 |
+
*,
|
| 167 |
+
canonicalized: bool = False,
|
| 168 |
+
) -> TouchFeatures:
|
| 169 |
+
expected = (8, 2, 21, 21)
|
| 170 |
+
if pressure.ndim != 5 or tuple(pressure.shape[1:]) != expected:
|
| 171 |
+
raise ValueError(
|
| 172 |
+
f"pressure must be [B,8,2,21,21], got {tuple(pressure.shape)}"
|
| 173 |
+
)
|
| 174 |
+
b = pressure.shape[0]
|
| 175 |
+
if temporal_valid is None:
|
| 176 |
+
temporal_valid = torch.ones(
|
| 177 |
+
b, 8, 2, dtype=torch.bool, device=pressure.device
|
| 178 |
+
)
|
| 179 |
+
if tuple(temporal_valid.shape) != (b, 8, 2):
|
| 180 |
+
raise ValueError(
|
| 181 |
+
f"temporal_valid must be {(b, 8, 2)}, got {tuple(temporal_valid.shape)}"
|
| 182 |
+
)
|
| 183 |
+
|
| 184 |
+
# Keep the most recent causal frames. [B,T,H,W] for each hand becomes
|
| 185 |
+
# [B*2,T,H,W], ordered L0,R0,L1,R1,...
|
| 186 |
+
pressure = pressure[:, -self.history_frames :]
|
| 187 |
+
temporal_valid = temporal_valid[:, -self.history_frames :]
|
| 188 |
+
flat = pressure.permute(0, 2, 1, 3, 4).reshape(
|
| 189 |
+
b * 2, self.history_frames, 21, 21
|
| 190 |
+
)
|
| 191 |
+
valid_flat = temporal_valid.permute(0, 2, 1).reshape(
|
| 192 |
+
b * 2, self.history_frames
|
| 193 |
+
)
|
| 194 |
+
flat = flat * valid_flat[:, :, None, None].to(flat.dtype)
|
| 195 |
+
side = torch.arange(2, device=pressure.device).repeat(b)
|
| 196 |
+
if not canonicalized:
|
| 197 |
+
_, _, _, canonicalize = _selected_encoder_api()
|
| 198 |
+
flat = canonicalize(flat, side)
|
| 199 |
+
pressure_masks = getattr(self.encoder, "pressure_masks", None)
|
| 200 |
+
if pressure_masks is not None:
|
| 201 |
+
spatial_valid = pressure_masks[side.to(pressure_masks.device)]
|
| 202 |
+
flat = flat * spatial_valid[:, None].to(
|
| 203 |
+
device=flat.device, dtype=flat.dtype
|
| 204 |
+
)
|
| 205 |
+
|
| 206 |
+
tokens = self.encoder(flat, side).reshape(b, 2, -1)
|
| 207 |
+
hand_valid = valid_flat.any(dim=1).reshape(b, 2)
|
| 208 |
+
tokens = tokens * hand_valid.unsqueeze(-1).to(tokens.dtype)
|
| 209 |
+
return TouchFeatures(tokens=tokens, valid=hand_valid)
|
model/tachin_paired30_cache_20260923/code/src/tactile_editor/full_cache.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Shared reader for the sharded full EgoTouch editor replay cache."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import bisect
|
| 6 |
+
import json
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
|
| 9 |
+
import numpy as np
|
| 10 |
+
import torch
|
| 11 |
+
from torch.utils.data import Dataset
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
ARRAY_NAMES = (
|
| 15 |
+
"base_action",
|
| 16 |
+
"target_action",
|
| 17 |
+
"action_mask",
|
| 18 |
+
"touch_pressure",
|
| 19 |
+
"touch_valid",
|
| 20 |
+
"decision_seconds",
|
| 21 |
+
"recording",
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class ShardedEditorDataset(Dataset):
|
| 26 |
+
"""Memory-map complete replay shards without copying the full cache."""
|
| 27 |
+
|
| 28 |
+
def __init__(
|
| 29 |
+
self,
|
| 30 |
+
cache_root: str | Path,
|
| 31 |
+
split: str,
|
| 32 |
+
*,
|
| 33 |
+
expected_shards: int = 8,
|
| 34 |
+
) -> None:
|
| 35 |
+
self.cache_root = Path(cache_root)
|
| 36 |
+
self.split = split
|
| 37 |
+
self.shards: list[tuple[int, dict[str, np.ndarray]]] = []
|
| 38 |
+
self.ends: list[int] = []
|
| 39 |
+
total = 0
|
| 40 |
+
for root in sorted(self.cache_root.glob("shard_*")):
|
| 41 |
+
metadata_path = root / split / "metadata.json"
|
| 42 |
+
if not metadata_path.is_file():
|
| 43 |
+
continue
|
| 44 |
+
metadata = json.loads(metadata_path.read_text())
|
| 45 |
+
count = int(metadata["valid_samples"])
|
| 46 |
+
arrays = {
|
| 47 |
+
name: np.load(root / split / f"{name}.npy", mmap_mode="r")
|
| 48 |
+
for name in ARRAY_NAMES
|
| 49 |
+
}
|
| 50 |
+
if any(len(value) < count for value in arrays.values()):
|
| 51 |
+
raise ValueError(f"shard arrays shorter than metadata: {root / split}")
|
| 52 |
+
self.shards.append((count, arrays))
|
| 53 |
+
total += count
|
| 54 |
+
self.ends.append(total)
|
| 55 |
+
if len(self.shards) != expected_shards:
|
| 56 |
+
raise RuntimeError(
|
| 57 |
+
f"expected {expected_shards} complete cache shards, found {len(self.shards)}"
|
| 58 |
+
)
|
| 59 |
+
|
| 60 |
+
def __len__(self) -> int:
|
| 61 |
+
return self.ends[-1]
|
| 62 |
+
|
| 63 |
+
def __getitem__(self, index: int) -> dict[str, torch.Tensor]:
|
| 64 |
+
if index < 0:
|
| 65 |
+
index += len(self)
|
| 66 |
+
if not 0 <= index < len(self):
|
| 67 |
+
raise IndexError(index)
|
| 68 |
+
shard = bisect.bisect_right(self.ends, index)
|
| 69 |
+
start = 0 if shard == 0 else self.ends[shard - 1]
|
| 70 |
+
row = index - start
|
| 71 |
+
arrays = self.shards[shard][1]
|
| 72 |
+
result = {
|
| 73 |
+
name: torch.from_numpy(np.array(value[row], copy=True))
|
| 74 |
+
for name, value in arrays.items()
|
| 75 |
+
}
|
| 76 |
+
result["index"] = torch.tensor(index, dtype=torch.long)
|
| 77 |
+
return result
|
| 78 |
+
|
| 79 |
+
def read_column(self, name: str) -> np.ndarray:
|
| 80 |
+
"""Materialize one valid column in global dataset order."""
|
| 81 |
+
|
| 82 |
+
if name not in ARRAY_NAMES:
|
| 83 |
+
raise KeyError(name)
|
| 84 |
+
return np.concatenate(
|
| 85 |
+
[np.asarray(arrays[name][:count]) for count, arrays in self.shards], axis=0
|
| 86 |
+
)
|
model/tachin_paired30_cache_20260923/code/src/tactile_editor/fusion.py
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""V1/V2/V3 fusion modules. V0 owns no tactile parameters."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from dataclasses import dataclass
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
from torch import nn
|
| 9 |
+
|
| 10 |
+
from .config import FusionConfig, Variant
|
| 11 |
+
from .encoder import TouchFeatures
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def _safe_key_padding(valid: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
| 15 |
+
"""Return an MHA mask without all-masked rows and the original row validity."""
|
| 16 |
+
|
| 17 |
+
row_valid = valid.any(dim=1)
|
| 18 |
+
key_padding = ~valid.to(torch.bool)
|
| 19 |
+
if bool((~row_valid).any()):
|
| 20 |
+
key_padding = key_padding.clone()
|
| 21 |
+
key_padding[~row_valid, 0] = False
|
| 22 |
+
return key_padding, row_valid
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class GatedCrossAttention(nn.Module):
|
| 26 |
+
"""Residual cross-attention that is an exact identity at initialization."""
|
| 27 |
+
|
| 28 |
+
def __init__(self, hidden_dim: int, num_heads: int) -> None:
|
| 29 |
+
super().__init__()
|
| 30 |
+
self.query_norm = nn.LayerNorm(hidden_dim)
|
| 31 |
+
self.memory_norm = nn.LayerNorm(hidden_dim)
|
| 32 |
+
self.attention = nn.MultiheadAttention(hidden_dim, num_heads, batch_first=True)
|
| 33 |
+
self.gate = nn.Parameter(torch.zeros(()))
|
| 34 |
+
|
| 35 |
+
def forward(
|
| 36 |
+
self, query: torch.Tensor, memory: torch.Tensor, memory_valid: torch.Tensor
|
| 37 |
+
) -> torch.Tensor:
|
| 38 |
+
key_padding, row_valid = _safe_key_padding(memory_valid)
|
| 39 |
+
delta, _ = self.attention(
|
| 40 |
+
self.query_norm(query),
|
| 41 |
+
self.memory_norm(memory),
|
| 42 |
+
self.memory_norm(memory),
|
| 43 |
+
key_padding_mask=key_padding,
|
| 44 |
+
need_weights=False,
|
| 45 |
+
)
|
| 46 |
+
delta = delta * row_valid[:, None, None].to(delta.dtype)
|
| 47 |
+
return query + torch.tanh(self.gate) * delta
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
class PosthocResidualEditor(nn.Module):
|
| 51 |
+
"""V1: correct a completed normalized action chunk with current touch."""
|
| 52 |
+
|
| 53 |
+
def __init__(self, cfg: FusionConfig, tactile_dim: int = 64) -> None:
|
| 54 |
+
super().__init__()
|
| 55 |
+
h = cfg.posthoc_hidden_dim
|
| 56 |
+
self.cfg = cfg
|
| 57 |
+
self.action_projection = nn.Linear(cfg.action_dim, h)
|
| 58 |
+
self.touch_projection = nn.Linear(tactile_dim, h)
|
| 59 |
+
self.hand_embedding = nn.Parameter(torch.zeros(1, 2, h))
|
| 60 |
+
nn.init.normal_(self.hand_embedding, std=0.02)
|
| 61 |
+
self.position = nn.Parameter(torch.zeros(1, cfg.chunk_size, h))
|
| 62 |
+
nn.init.normal_(self.position, std=0.02)
|
| 63 |
+
self.touch_attention = GatedCrossAttention(h, cfg.posthoc_heads)
|
| 64 |
+
nn.init.constant_(self.touch_attention.gate, cfg.posthoc_touch_gate_init)
|
| 65 |
+
layer = nn.TransformerEncoderLayer(
|
| 66 |
+
h,
|
| 67 |
+
cfg.posthoc_heads,
|
| 68 |
+
dim_feedforward=4 * h,
|
| 69 |
+
batch_first=True,
|
| 70 |
+
norm_first=True,
|
| 71 |
+
)
|
| 72 |
+
self.trunk = nn.TransformerEncoder(
|
| 73 |
+
layer, num_layers=cfg.posthoc_layers, enable_nested_tensor=False
|
| 74 |
+
)
|
| 75 |
+
self.delta_head = nn.Linear(h, cfg.action_dim)
|
| 76 |
+
self.edit_gate = nn.Linear(h, cfg.action_dim)
|
| 77 |
+
nn.init.zeros_(self.delta_head.weight)
|
| 78 |
+
nn.init.zeros_(self.delta_head.bias)
|
| 79 |
+
nn.init.zeros_(self.edit_gate.weight)
|
| 80 |
+
nn.init.constant_(self.edit_gate.bias, cfg.posthoc_edit_gate_bias)
|
| 81 |
+
|
| 82 |
+
def forward(
|
| 83 |
+
self,
|
| 84 |
+
base_action: torch.Tensor,
|
| 85 |
+
touch: TouchFeatures,
|
| 86 |
+
edit_mask: torch.Tensor,
|
| 87 |
+
) -> tuple[torch.Tensor, dict[str, torch.Tensor]]:
|
| 88 |
+
expected = (self.cfg.chunk_size, self.cfg.action_dim)
|
| 89 |
+
if base_action.ndim != 3 or tuple(base_action.shape[1:]) != expected:
|
| 90 |
+
raise ValueError(f"base_action must be [B,{expected[0]},{expected[1]}]")
|
| 91 |
+
if edit_mask.shape != base_action.shape:
|
| 92 |
+
raise ValueError("edit_mask must have the same shape as base_action")
|
| 93 |
+
action = self.action_projection(base_action) + self.position
|
| 94 |
+
if self.cfg.posthoc_use_touch:
|
| 95 |
+
tactile = self.touch_projection(touch.tokens) + self.hand_embedding
|
| 96 |
+
action = self.touch_attention(action, tactile, touch.valid)
|
| 97 |
+
hidden = self.trunk(action)
|
| 98 |
+
raw_delta = self.delta_head(hidden)
|
| 99 |
+
gate = torch.sigmoid(self.edit_gate(hidden))
|
| 100 |
+
# The post-hoc editor is also the action-only control path when a touch
|
| 101 |
+
# frame is unavailable. ``GatedCrossAttention`` already returns the
|
| 102 |
+
# unchanged action stream for an all-invalid tactile row, so gating the
|
| 103 |
+
# residual a second time here would silently disable the control model
|
| 104 |
+
# and make missing-touch samples contribute no learning signal.
|
| 105 |
+
effective = raw_delta * gate * edit_mask.to(raw_delta.dtype)
|
| 106 |
+
return base_action + effective, {
|
| 107 |
+
"raw_delta": raw_delta,
|
| 108 |
+
"gate": gate,
|
| 109 |
+
"effective_delta": effective,
|
| 110 |
+
}
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
class CenteredArticulationTouchEditor(nn.Module):
|
| 114 |
+
"""Legacy centered touch-only editor kept for prior ablation checkpoints."""
|
| 115 |
+
|
| 116 |
+
ARTICULATION_GROUPS = ((6, 69), (75, 138))
|
| 117 |
+
|
| 118 |
+
def __init__(self, cfg: FusionConfig, tactile_dim: int = 64) -> None:
|
| 119 |
+
super().__init__()
|
| 120 |
+
if cfg.action_dim < self.ARTICULATION_GROUPS[-1][1]:
|
| 121 |
+
raise ValueError("action_dim is too small for bimanual MANO articulation")
|
| 122 |
+
if getattr(cfg, "posthoc_dropout", 0.0) != 0.0:
|
| 123 |
+
raise ValueError("centered subtraction requires posthoc_dropout=0")
|
| 124 |
+
self.cfg = cfg
|
| 125 |
+
self.residual = PosthocResidualEditor(cfg, tactile_dim=tactile_dim)
|
| 126 |
+
component_mask = torch.zeros(1, 1, cfg.action_dim, dtype=torch.bool)
|
| 127 |
+
for start, stop in self.ARTICULATION_GROUPS:
|
| 128 |
+
component_mask[..., start:stop] = True
|
| 129 |
+
self.register_buffer("component_mask", component_mask, persistent=False)
|
| 130 |
+
|
| 131 |
+
def forward(
|
| 132 |
+
self,
|
| 133 |
+
calibrated_action: torch.Tensor,
|
| 134 |
+
touch: TouchFeatures,
|
| 135 |
+
edit_mask: torch.Tensor,
|
| 136 |
+
) -> tuple[torch.Tensor, dict[str, torch.Tensor]]:
|
| 137 |
+
_, matched = self.residual(calibrated_action, touch, edit_mask)
|
| 138 |
+
reference = TouchFeatures(
|
| 139 |
+
tokens=torch.zeros_like(touch.tokens),
|
| 140 |
+
valid=torch.zeros_like(touch.valid),
|
| 141 |
+
)
|
| 142 |
+
_, action_only = self.residual(calibrated_action, reference, edit_mask)
|
| 143 |
+
has_touch = touch.valid.any(dim=1)[:, None, None]
|
| 144 |
+
allowed = edit_mask.bool() & self.component_mask & has_touch
|
| 145 |
+
touch_delta = matched["effective_delta"] - action_only["effective_delta"]
|
| 146 |
+
effective = touch_delta * allowed.to(touch_delta.dtype)
|
| 147 |
+
return calibrated_action + effective, {
|
| 148 |
+
"matched_delta": matched["effective_delta"],
|
| 149 |
+
"action_only_delta": action_only["effective_delta"],
|
| 150 |
+
"effective_delta": effective,
|
| 151 |
+
"allowed_mask": allowed,
|
| 152 |
+
}
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
@dataclass
|
| 156 |
+
class ExpertFusionContext:
|
| 157 |
+
touch: torch.Tensor
|
| 158 |
+
touch_valid: torch.Tensor
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
class ExpertTouchFusion(nn.Module):
|
| 162 |
+
"""V2/V3: inject touch into selected layers of Method D's action stream."""
|
| 163 |
+
|
| 164 |
+
def __init__(
|
| 165 |
+
self, variant: Variant, cfg: FusionConfig, tactile_dim: int = 64
|
| 166 |
+
) -> None:
|
| 167 |
+
super().__init__()
|
| 168 |
+
if variant not in {Variant.V2_EXPERT_TOUCH, Variant.V3_MEMORY_TOUCH}:
|
| 169 |
+
raise ValueError(f"expert fusion does not implement {variant.value}")
|
| 170 |
+
self.variant = variant
|
| 171 |
+
self.cfg = cfg
|
| 172 |
+
self.touch_projection = nn.Linear(tactile_dim, cfg.hidden_dim)
|
| 173 |
+
self.hand_embedding = nn.Parameter(torch.zeros(1, 2, cfg.hidden_dim))
|
| 174 |
+
nn.init.normal_(self.hand_embedding, std=0.02)
|
| 175 |
+
self.action_from_touch = GatedCrossAttention(cfg.hidden_dim, cfg.num_heads)
|
| 176 |
+
self.touch_from_memory = (
|
| 177 |
+
nn.MultiheadAttention(cfg.hidden_dim, cfg.num_heads, batch_first=True)
|
| 178 |
+
if variant == Variant.V3_MEMORY_TOUCH
|
| 179 |
+
else None
|
| 180 |
+
)
|
| 181 |
+
self.touch_memory_norm = nn.LayerNorm(cfg.hidden_dim)
|
| 182 |
+
|
| 183 |
+
def prepare(
|
| 184 |
+
self, touch: TouchFeatures, expert_memory: torch.Tensor
|
| 185 |
+
) -> ExpertFusionContext:
|
| 186 |
+
tactile = self.touch_projection(touch.tokens) + self.hand_embedding
|
| 187 |
+
tactile = tactile * touch.valid.unsqueeze(-1).to(tactile.dtype)
|
| 188 |
+
if self.touch_from_memory is not None:
|
| 189 |
+
query = self.touch_memory_norm(tactile)
|
| 190 |
+
conditioned, _ = self.touch_from_memory(
|
| 191 |
+
query, expert_memory, expert_memory, need_weights=False
|
| 192 |
+
)
|
| 193 |
+
tactile = (tactile + conditioned) * touch.valid.unsqueeze(-1).to(
|
| 194 |
+
tactile.dtype
|
| 195 |
+
)
|
| 196 |
+
return ExpertFusionContext(touch=tactile, touch_valid=touch.valid)
|
| 197 |
+
|
| 198 |
+
def forward(
|
| 199 |
+
self, layer_index: int, action_hidden: torch.Tensor, ctx: ExpertFusionContext
|
| 200 |
+
) -> torch.Tensor:
|
| 201 |
+
if layer_index not in self.cfg.injection_layers:
|
| 202 |
+
return action_hidden
|
| 203 |
+
return self.action_from_touch(action_hidden, ctx.touch, ctx.touch_valid)
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
def build_fusion(variant: Variant, cfg: FusionConfig) -> nn.Module | None:
|
| 207 |
+
if variant == Variant.V0_BASE:
|
| 208 |
+
return None
|
| 209 |
+
if variant == Variant.V1_POSTHOC:
|
| 210 |
+
return PosthocResidualEditor(cfg)
|
| 211 |
+
return ExpertTouchFusion(variant, cfg)
|
model/tachin_paired30_cache_20260923/code/src/tactile_editor/informative_data.py
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Leakage-safe selection of samples with active-hand tactile events."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
|
| 7 |
+
from tactile_editor.architecture_ablation import tactile_contact_statistics
|
| 8 |
+
from tactile_editor.training import build_future_action_mask
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
LEFT_ARTICULATION = (6, 69)
|
| 12 |
+
RIGHT_ARTICULATION = (75, 138)
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def supervised_hands(
|
| 16 |
+
action_mask: torch.Tensor,
|
| 17 |
+
decision_seconds: torch.Tensor,
|
| 18 |
+
) -> torch.Tensor:
|
| 19 |
+
"""Return the hands that have editable articulation after each decision."""
|
| 20 |
+
|
| 21 |
+
future = build_future_action_mask(
|
| 22 |
+
action_mask.bool(), decision_seconds.to(dtype=torch.float64)
|
| 23 |
+
)
|
| 24 |
+
left = future[..., LEFT_ARTICULATION[0] : LEFT_ARTICULATION[1]].any((1, 2))
|
| 25 |
+
right = future[..., RIGHT_ARTICULATION[0] : RIGHT_ARTICULATION[1]].any((1, 2))
|
| 26 |
+
return torch.stack((left, right), dim=1).cpu()
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def tactile_event_change_per_hand(
|
| 30 |
+
touch_pressure: torch.Tensor,
|
| 31 |
+
touch_valid: torch.Tensor,
|
| 32 |
+
pressure_masks: torch.Tensor,
|
| 33 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 34 |
+
"""Return latest-pair pressure change and validity for each hand."""
|
| 35 |
+
|
| 36 |
+
if touch_pressure.ndim != 5 or tuple(touch_pressure.shape[1:]) != (8, 2, 21, 21):
|
| 37 |
+
raise ValueError("touch_pressure must be [N,8,2,21,21]")
|
| 38 |
+
if tuple(touch_valid.shape) != tuple(touch_pressure.shape[:3]):
|
| 39 |
+
raise ValueError("touch_valid must be [N,8,2]")
|
| 40 |
+
if tuple(pressure_masks.shape) != (2, 21, 21):
|
| 41 |
+
raise ValueError("pressure_masks must be [2,21,21]")
|
| 42 |
+
pressure = torch.nan_to_num(touch_pressure.detach().cpu().float()).clone()
|
| 43 |
+
pressure[:, :, 0] = pressure[:, :, 0].flip(-1)
|
| 44 |
+
masks = pressure_masks.detach().cpu().bool()
|
| 45 |
+
changes = []
|
| 46 |
+
for hand in range(2):
|
| 47 |
+
cells = pressure[:, -2:, hand, masks[hand]]
|
| 48 |
+
changes.append((cells[:, 1] - cells[:, 0]).abs().mean(-1))
|
| 49 |
+
pair_valid = touch_valid.detach().cpu().bool()[:, -2:].all(1)
|
| 50 |
+
change = torch.stack(changes, dim=1).masked_fill(~pair_valid, 0)
|
| 51 |
+
return change, pair_valid
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def tactile_history_dynamics_per_hand(
|
| 55 |
+
touch_pressure: torch.Tensor,
|
| 56 |
+
touch_valid: torch.Tensor,
|
| 57 |
+
pressure_masks: torch.Tensor,
|
| 58 |
+
*,
|
| 59 |
+
contact_threshold: float = 0.1,
|
| 60 |
+
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 61 |
+
"""Return change, transition, pair validity, and contact for each hand."""
|
| 62 |
+
|
| 63 |
+
if touch_pressure.ndim != 5 or tuple(touch_pressure.shape[1:]) != (8, 2, 21, 21):
|
| 64 |
+
raise ValueError("touch_pressure must be [N,8,2,21,21]")
|
| 65 |
+
if tuple(touch_valid.shape) != tuple(touch_pressure.shape[:3]):
|
| 66 |
+
raise ValueError("touch_valid must be [N,8,2]")
|
| 67 |
+
if tuple(pressure_masks.shape) != (2, 21, 21):
|
| 68 |
+
raise ValueError("pressure_masks must be [2,21,21]")
|
| 69 |
+
pressure = torch.nan_to_num(touch_pressure.detach().cpu().float()).clone()
|
| 70 |
+
pressure[:, :, 0] = pressure[:, :, 0].flip(-1)
|
| 71 |
+
valid = touch_valid.detach().cpu().bool()
|
| 72 |
+
masks = pressure_masks.detach().cpu().bool()
|
| 73 |
+
maximum_changes = []
|
| 74 |
+
transitions = []
|
| 75 |
+
has_pairs = []
|
| 76 |
+
has_contacts = []
|
| 77 |
+
for hand in range(2):
|
| 78 |
+
cells = pressure[:, :, hand, masks[hand]]
|
| 79 |
+
pair_valid = valid[:, 1:, hand] & valid[:, :-1, hand]
|
| 80 |
+
changes = (cells[:, 1:] - cells[:, :-1]).abs().mean(-1)
|
| 81 |
+
maximum = changes.masked_fill(~pair_valid, -torch.inf).max(1).values
|
| 82 |
+
has_pair = pair_valid.any(1)
|
| 83 |
+
maximum = maximum.masked_fill(~has_pair, 0)
|
| 84 |
+
contact = cells.topk(5, dim=-1).values.mean(-1) >= contact_threshold
|
| 85 |
+
transition = ((contact[:, 1:] != contact[:, :-1]) & pair_valid).any(1)
|
| 86 |
+
maximum_changes.append(maximum)
|
| 87 |
+
transitions.append(transition)
|
| 88 |
+
has_pairs.append(has_pair)
|
| 89 |
+
has_contacts.append((contact & valid[:, :, hand]).any(1))
|
| 90 |
+
return (
|
| 91 |
+
torch.stack(maximum_changes, dim=1),
|
| 92 |
+
torch.stack(transitions, dim=1),
|
| 93 |
+
torch.stack(has_pairs, dim=1),
|
| 94 |
+
torch.stack(has_contacts, dim=1),
|
| 95 |
+
)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def active_hand_selection(
|
| 99 |
+
touch_pressure: torch.Tensor,
|
| 100 |
+
touch_valid: torch.Tensor,
|
| 101 |
+
action_mask: torch.Tensor,
|
| 102 |
+
decision_seconds: torch.Tensor,
|
| 103 |
+
pressure_masks: torch.Tensor,
|
| 104 |
+
*,
|
| 105 |
+
pressure_change_threshold: float,
|
| 106 |
+
contact_threshold: float = 0.1,
|
| 107 |
+
) -> dict[str, torch.Tensor]:
|
| 108 |
+
"""Classify rows without reading the target action or its prediction error."""
|
| 109 |
+
|
| 110 |
+
active = supervised_hands(action_mask, decision_seconds)
|
| 111 |
+
contact = tactile_contact_statistics(
|
| 112 |
+
touch_pressure, touch_valid, pressure_masks
|
| 113 |
+
)
|
| 114 |
+
change, pair_valid = tactile_event_change_per_hand(
|
| 115 |
+
touch_pressure, touch_valid, pressure_masks
|
| 116 |
+
)
|
| 117 |
+
now_contact = contact["current_score_per_hand"] >= contact_threshold
|
| 118 |
+
previous_contact = contact["previous_score_per_hand"] >= contact_threshold
|
| 119 |
+
active_pair = active & pair_valid
|
| 120 |
+
transition_per_hand = (now_contact != previous_contact) & active_pair
|
| 121 |
+
changing_per_hand = (change >= pressure_change_threshold) & active_pair
|
| 122 |
+
dynamic_per_hand = transition_per_hand | changing_per_hand
|
| 123 |
+
contact_per_hand = (now_contact | previous_contact) & active_pair
|
| 124 |
+
supervised = active.any(1)
|
| 125 |
+
dynamic = dynamic_per_hand.any(1)
|
| 126 |
+
stable_contact = contact_per_hand.any(1) & ~dynamic
|
| 127 |
+
low_or_no_contact = active_pair.any(1) & ~contact_per_hand.any(1) & ~dynamic
|
| 128 |
+
return {
|
| 129 |
+
"supervised": supervised,
|
| 130 |
+
"active_hands": active,
|
| 131 |
+
"active_pair_valid": active_pair,
|
| 132 |
+
"active_event_change": change,
|
| 133 |
+
"active_transition_per_hand": transition_per_hand,
|
| 134 |
+
"active_changing_per_hand": changing_per_hand,
|
| 135 |
+
"active_dynamic": dynamic,
|
| 136 |
+
"active_stable_contact": stable_contact,
|
| 137 |
+
"active_low_or_no_contact": low_or_no_contact,
|
| 138 |
+
}
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def active_hand_history_selection(
|
| 142 |
+
touch_pressure: torch.Tensor,
|
| 143 |
+
touch_valid: torch.Tensor,
|
| 144 |
+
action_mask: torch.Tensor,
|
| 145 |
+
decision_seconds: torch.Tensor,
|
| 146 |
+
pressure_masks: torch.Tensor,
|
| 147 |
+
*,
|
| 148 |
+
pressure_change_threshold: float,
|
| 149 |
+
contact_threshold: float = 0.1,
|
| 150 |
+
) -> dict[str, torch.Tensor]:
|
| 151 |
+
"""Classify rows from the full causal history of the supervised hand."""
|
| 152 |
+
|
| 153 |
+
active = supervised_hands(action_mask, decision_seconds)
|
| 154 |
+
maximum_change, transition, has_pair, has_contact = tactile_history_dynamics_per_hand(
|
| 155 |
+
touch_pressure,
|
| 156 |
+
touch_valid,
|
| 157 |
+
pressure_masks,
|
| 158 |
+
contact_threshold=contact_threshold,
|
| 159 |
+
)
|
| 160 |
+
active_pair = active & has_pair
|
| 161 |
+
changing = (maximum_change >= pressure_change_threshold) & active_pair
|
| 162 |
+
dynamic_per_hand = (transition & active_pair) | changing
|
| 163 |
+
dynamic = dynamic_per_hand.any(1)
|
| 164 |
+
contact_per_hand = has_contact & active_pair
|
| 165 |
+
stable_contact = contact_per_hand.any(1) & ~dynamic
|
| 166 |
+
low_or_no_contact = active_pair.any(1) & ~contact_per_hand.any(1) & ~dynamic
|
| 167 |
+
return {
|
| 168 |
+
"supervised": active.any(1),
|
| 169 |
+
"active_hands": active,
|
| 170 |
+
"active_pair_valid": active_pair,
|
| 171 |
+
"active_event_change": maximum_change,
|
| 172 |
+
"active_transition_per_hand": transition & active_pair,
|
| 173 |
+
"active_changing_per_hand": changing,
|
| 174 |
+
"active_dynamic": dynamic,
|
| 175 |
+
"active_stable_contact": stable_contact,
|
| 176 |
+
"active_low_or_no_contact": low_or_no_contact,
|
| 177 |
+
}
|
model/tachin_paired30_cache_20260923/code/src/tactile_editor/memoryvla_baseline.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Restore human MemoryVLA policies or explicitly initialize a new human policy."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import copy
|
| 5 |
+
import hashlib
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def tokenizer_fingerprint(directory):
|
| 12 |
+
"""Identify the exact config and tokenizer assets used by the frozen VLM."""
|
| 13 |
+
root = Path(directory)
|
| 14 |
+
files = sorted(p for p in root.iterdir() if p.is_file() and
|
| 15 |
+
(p.name.startswith('tokenizer') or p.name in ('config.json', 'special_tokens_map.json')))
|
| 16 |
+
if not files or not (root / 'config.json').is_file():
|
| 17 |
+
raise FileNotFoundError(f"missing Llama config/tokenizer files in {root}")
|
| 18 |
+
return {p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in files}
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def validate_action_stats(model):
|
| 22 |
+
if not bool(model.stats_loaded):
|
| 23 |
+
raise ValueError("MemoryVLA checkpoint has no human action normalization")
|
| 24 |
+
for name in ('action_mean', 'action_std'):
|
| 25 |
+
value = getattr(model, name)
|
| 26 |
+
if value.shape != (192,) or not torch.isfinite(value).all():
|
| 27 |
+
raise ValueError(f"invalid MemoryVLA {name}")
|
| 28 |
+
if (model.action_std <= 0).any():
|
| 29 |
+
raise ValueError("MemoryVLA action_std must be positive")
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def initialize_memoryvla(pretrained_checkpoint, llama_directory, *, action_mean, action_std,
|
| 33 |
+
overrides=None, device='cpu', seed=0):
|
| 34 |
+
"""Initialize human I/O from CogACT or native MemoryVLA, ready for base training.
|
| 35 |
+
|
| 36 |
+
New I/O weights have not learned human actions. This is not a Table 1 base.
|
| 37 |
+
Statistics must be supplied from human training data (synthetic only for smoke tests).
|
| 38 |
+
"""
|
| 39 |
+
from ._baselines.memoryvla_official.config import from_json
|
| 40 |
+
from ._baselines.memoryvla_official.model import MemoryVLAModel, build_tokenizer
|
| 41 |
+
values = copy.deepcopy(overrides or {})
|
| 42 |
+
values.setdefault('vlm', {}).update(pretrained_checkpoint=str(pretrained_checkpoint),
|
| 43 |
+
llama_local_dir=str(llama_directory), freeze_vlm=True)
|
| 44 |
+
values.setdefault('data', {})['stats_path'] = 'provided://human-training-statistics'
|
| 45 |
+
cfg = from_json(values)
|
| 46 |
+
# Validate assets and prompt before allocating the large model.
|
| 47 |
+
assets = tokenizer_fingerprint(llama_directory)
|
| 48 |
+
tokenizer = build_tokenizer(cfg)
|
| 49 |
+
probe = tokenizer.prompt('Move the hand.')
|
| 50 |
+
expected = tokenizer.tok(probe.removesuffix(' </s>'), add_special_tokens=True).input_ids + [29871, 2]
|
| 51 |
+
if tokenizer.tok(probe, add_special_tokens=True).input_ids != expected:
|
| 52 |
+
raise ValueError('Llama tokenizer differs from the MemoryVLA EOS prompt convention')
|
| 53 |
+
with torch.random.fork_rng(devices=[]):
|
| 54 |
+
torch.manual_seed(seed)
|
| 55 |
+
model = MemoryVLAModel(cfg, load_weights=True)
|
| 56 |
+
model._table1_tokenizer = tokenizer
|
| 57 |
+
model.set_normalisation(action_mean=action_mean, action_std=action_std)
|
| 58 |
+
validate_action_stats(model)
|
| 59 |
+
model.trainable_params_setup()
|
| 60 |
+
model.to(device)
|
| 61 |
+
model._human_action_trained = False
|
| 62 |
+
info = {'model_id': 'memoryvla', 'layout': 'keypoints', 'human_action_trained': False,
|
| 63 |
+
'initialization_seed': seed, 'pretrained_checkpoint': str(Path(pretrained_checkpoint).resolve()),
|
| 64 |
+
'tokenizer_assets': assets, 'load_report': model.load_report}
|
| 65 |
+
return model, cfg, info
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def load_memoryvla_human_checkpoint(checkpoint, *, pretrained_checkpoint, llama_directory,
|
| 69 |
+
device='cpu'):
|
| 70 |
+
"""Restore collaborator checkpoints, including the separately stored frozen VLM.
|
| 71 |
+
|
| 72 |
+
The original VLM checkpoint must match the one used during base training.
|
| 73 |
+
All saved trainable parameters and normalization buffers must match exactly.
|
| 74 |
+
"""
|
| 75 |
+
payload = torch.load(checkpoint, map_location='cpu', mmap=True, weights_only=False)
|
| 76 |
+
run = payload['config']
|
| 77 |
+
if run.get('model') != 'memoryvla_official':
|
| 78 |
+
raise ValueError('expected a collaborator memoryvla_official human checkpoint')
|
| 79 |
+
if payload.get('human_action_trained') is False:
|
| 80 |
+
raise ValueError('untrained human I/O cannot serve as a Table 1 baseline')
|
| 81 |
+
sd = payload['model']
|
| 82 |
+
if not bool(sd.get('stats_loaded', False)):
|
| 83 |
+
raise ValueError('checkpoint has no human action normalization')
|
| 84 |
+
model, cfg, info = initialize_memoryvla(
|
| 85 |
+
pretrained_checkpoint, llama_directory, action_mean=sd['action_mean'], action_std=sd['action_std'],
|
| 86 |
+
overrides=run.get('config', {}), device='cpu')
|
| 87 |
+
model.load_state_dict(sd, strict=True)
|
| 88 |
+
validate_action_stats(model)
|
| 89 |
+
for name in ('action_mean', 'action_std'):
|
| 90 |
+
if name in payload:
|
| 91 |
+
reference = torch.as_tensor(payload[name], dtype=torch.float32)
|
| 92 |
+
if not torch.isfinite(reference).all() or (name == 'action_std' and (reference < 0).any()):
|
| 93 |
+
raise ValueError(f'invalid top-level {name}')
|
| 94 |
+
# The trainer saves raw statistics above, but clamps constant channels
|
| 95 |
+
# to 1e-6 when installing the model buffers.
|
| 96 |
+
if name == 'action_std':
|
| 97 |
+
reference = reference.clamp_min(1e-6)
|
| 98 |
+
torch.testing.assert_close(getattr(model, name), reference, rtol=0, atol=0)
|
| 99 |
+
model.requires_grad_(False).eval().to(device)
|
| 100 |
+
model._human_action_trained = True
|
| 101 |
+
info.update(checkpoint=str(Path(checkpoint).resolve()), step=int(payload['step']),
|
| 102 |
+
human_action_trained=True, strict_load=True, frozen_vlm_external=True,
|
| 103 |
+
state_tensors=len(sd))
|
| 104 |
+
return model, cfg, info
|
model/tachin_paired30_cache_20260923/code/src/tactile_editor/model_adapters.py
ADDED
|
@@ -0,0 +1,287 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Model adapters that normalize heterogeneous VLA outputs to BaseActionContext.
|
| 2 |
+
|
| 3 |
+
The adapters intentionally do not know dataset layouts. A dataset/inference
|
| 4 |
+
loader supplies the observation object; the adapter only calls the model and
|
| 5 |
+
normalizes its returned action/memory fields.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
from collections.abc import Callable, Mapping
|
| 11 |
+
from dataclasses import dataclass, replace
|
| 12 |
+
from typing import Any
|
| 13 |
+
|
| 14 |
+
import torch
|
| 15 |
+
|
| 16 |
+
from .action_layout import ACTION_DIM, CHUNK_SIZE, DEFAULT_ACTION_LAYOUT
|
| 17 |
+
from .editor_interface import BaseActionContext
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
ActionPredictor = Callable[..., Any]
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
@dataclass(frozen=True)
|
| 24 |
+
class AdapterSpec:
|
| 25 |
+
"""Output convention for one base model checkpoint."""
|
| 26 |
+
|
| 27 |
+
model_id: str
|
| 28 |
+
normalization_id: str
|
| 29 |
+
output_is_normalized: bool = True
|
| 30 |
+
action_mean: torch.Tensor | None = None
|
| 31 |
+
action_std: torch.Tensor | None = None
|
| 32 |
+
action_key: str = "base_action"
|
| 33 |
+
mask_key: str = "action_mask"
|
| 34 |
+
memory_key: str = "memory_tokens"
|
| 35 |
+
image_key: str = "image_tokens"
|
| 36 |
+
|
| 37 |
+
def __post_init__(self) -> None:
|
| 38 |
+
if not self.model_id or not self.normalization_id:
|
| 39 |
+
raise ValueError("model_id and normalization_id are required")
|
| 40 |
+
if not self.output_is_normalized:
|
| 41 |
+
if self.action_mean is None or self.action_std is None:
|
| 42 |
+
raise ValueError("raw action adapters require action_mean and action_std")
|
| 43 |
+
for name, value in (("action_mean", self.action_mean), ("action_std", self.action_std)):
|
| 44 |
+
if value is not None and tuple(value.shape) != (ACTION_DIM,):
|
| 45 |
+
raise ValueError(f"{name} must have shape [192]")
|
| 46 |
+
if self.action_std is not None and (self.action_std <= 0).any():
|
| 47 |
+
raise ValueError("action_std must be positive")
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _first(mapping: Mapping[str, Any], names: tuple[str, ...]) -> Any:
|
| 51 |
+
for name in names:
|
| 52 |
+
if name in mapping:
|
| 53 |
+
return mapping[name]
|
| 54 |
+
return None
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
class ModelAdapter:
|
| 58 |
+
"""Call a model and convert its output to a validated BaseActionContext."""
|
| 59 |
+
|
| 60 |
+
ACTION_NAMES = ("base_action", "action", "actions", "action_chunk", "norm_action", "normalized_action")
|
| 61 |
+
MASK_NAMES = ("action_mask", "mask", "valid_mask")
|
| 62 |
+
MEMORY_NAMES = ("memory_tokens", "final_memory", "memory", "m_f")
|
| 63 |
+
IMAGE_NAMES = ("image_tokens", "vision_tokens", "image_features")
|
| 64 |
+
|
| 65 |
+
def __init__(
|
| 66 |
+
self,
|
| 67 |
+
model: Any,
|
| 68 |
+
spec: AdapterSpec,
|
| 69 |
+
*,
|
| 70 |
+
predictor: ActionPredictor | None = None,
|
| 71 |
+
action_converter: Callable[[torch.Tensor], torch.Tensor] | None = None,
|
| 72 |
+
method_names: tuple[str, ...] = (
|
| 73 |
+
"predict_base_action",
|
| 74 |
+
"sample_action_context",
|
| 75 |
+
"select_action_context",
|
| 76 |
+
"predict_action",
|
| 77 |
+
"sample_action",
|
| 78 |
+
"select_action",
|
| 79 |
+
),
|
| 80 |
+
) -> None:
|
| 81 |
+
self.model = model
|
| 82 |
+
self.spec = spec
|
| 83 |
+
self.predictor = predictor
|
| 84 |
+
self.action_converter = action_converter
|
| 85 |
+
self.method_names = method_names
|
| 86 |
+
|
| 87 |
+
def _predict(self, observation: Any, **model_kwargs: Any) -> Any:
|
| 88 |
+
if self.predictor is not None:
|
| 89 |
+
return self.predictor(observation, **model_kwargs)
|
| 90 |
+
if callable(self.model) and not any(hasattr(self.model, name) for name in self.method_names):
|
| 91 |
+
return self.model(observation, **model_kwargs)
|
| 92 |
+
for name in self.method_names:
|
| 93 |
+
method = getattr(self.model, name, None)
|
| 94 |
+
if method is not None:
|
| 95 |
+
return method(observation, **model_kwargs)
|
| 96 |
+
raise AttributeError(
|
| 97 |
+
f"{self.spec.model_id} adapter found no supported inference method; "
|
| 98 |
+
f"expected one of {self.method_names}"
|
| 99 |
+
)
|
| 100 |
+
|
| 101 |
+
def _unpack(self, result: Any) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None, dict[str, Any]]:
|
| 102 |
+
if isinstance(result, BaseActionContext):
|
| 103 |
+
return result.base_action, result.memory_tokens, result.image_tokens, {
|
| 104 |
+
"action_mask": result.action_mask,
|
| 105 |
+
"edit_mask": result.edit_mask,
|
| 106 |
+
"metadata": result.metadata,
|
| 107 |
+
}
|
| 108 |
+
if isinstance(result, Mapping):
|
| 109 |
+
action = _first(result, (self.spec.action_key, *self.ACTION_NAMES))
|
| 110 |
+
memory = _first(result, (self.spec.memory_key, *self.MEMORY_NAMES))
|
| 111 |
+
image = _first(result, (self.spec.image_key, *self.IMAGE_NAMES))
|
| 112 |
+
extras = dict(result)
|
| 113 |
+
return action, memory, image, extras
|
| 114 |
+
if isinstance(result, (tuple, list)):
|
| 115 |
+
if not result:
|
| 116 |
+
raise ValueError("model adapter received an empty output tuple")
|
| 117 |
+
action = result[0]
|
| 118 |
+
memory = result[1] if len(result) > 1 else None
|
| 119 |
+
image = result[2] if len(result) > 2 else None
|
| 120 |
+
return action, memory, image, {}
|
| 121 |
+
return result, None, None, {}
|
| 122 |
+
|
| 123 |
+
def __call__(
|
| 124 |
+
self,
|
| 125 |
+
observation: Any,
|
| 126 |
+
*,
|
| 127 |
+
action_mask: torch.Tensor | None = None,
|
| 128 |
+
edit_mask: torch.Tensor | None = None,
|
| 129 |
+
**model_kwargs: Any,
|
| 130 |
+
) -> BaseActionContext:
|
| 131 |
+
result = self._predict(observation, **model_kwargs)
|
| 132 |
+
action, memory, image, extras = self._unpack(result)
|
| 133 |
+
if action is None:
|
| 134 |
+
raise ValueError(
|
| 135 |
+
f"{self.spec.model_id} predictor did not return an action; "
|
| 136 |
+
f"expected one of {(self.spec.action_key, *self.ACTION_NAMES)}"
|
| 137 |
+
)
|
| 138 |
+
if not isinstance(action, torch.Tensor):
|
| 139 |
+
action = torch.as_tensor(action)
|
| 140 |
+
if self.action_converter is not None:
|
| 141 |
+
action = self.action_converter(action)
|
| 142 |
+
if action.ndim != 3 or tuple(action.shape[1:]) != (CHUNK_SIZE, ACTION_DIM):
|
| 143 |
+
raise ValueError(
|
| 144 |
+
f"{self.spec.model_id} must emit [B,{CHUNK_SIZE},{ACTION_DIM}], "
|
| 145 |
+
f"got {tuple(action.shape)}; provide an explicit action converter"
|
| 146 |
+
)
|
| 147 |
+
action = action.float()
|
| 148 |
+
if not self.spec.output_is_normalized:
|
| 149 |
+
mean = self.spec.action_mean.to(action.device, action.dtype)
|
| 150 |
+
std = self.spec.action_std.to(action.device, action.dtype)
|
| 151 |
+
action = (action - mean) / std
|
| 152 |
+
|
| 153 |
+
supplied_mask = _first(extras, (self.spec.mask_key, *self.MASK_NAMES))
|
| 154 |
+
mask = action_mask if action_mask is not None else supplied_mask
|
| 155 |
+
if mask is None:
|
| 156 |
+
mask = DEFAULT_ACTION_LAYOUT.physical_mask(
|
| 157 |
+
tuple(action.shape), device=action.device
|
| 158 |
+
)
|
| 159 |
+
mask = mask.to(device=action.device, dtype=torch.bool)
|
| 160 |
+
supplied_edit = _first(extras, ("edit_mask", "future_mask", "editable_mask"))
|
| 161 |
+
if edit_mask is None:
|
| 162 |
+
edit_mask = supplied_edit
|
| 163 |
+
return BaseActionContext(
|
| 164 |
+
base_action=action,
|
| 165 |
+
action_mask=mask,
|
| 166 |
+
edit_mask=edit_mask,
|
| 167 |
+
model_id=self.spec.model_id,
|
| 168 |
+
normalization_id=self.spec.normalization_id,
|
| 169 |
+
memory_tokens=memory,
|
| 170 |
+
image_tokens=image,
|
| 171 |
+
metadata={"adapter_output": extras},
|
| 172 |
+
)
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
class SmolVLAAdapter(ModelAdapter):
|
| 176 |
+
"""Adapter for the local V4-A/B/C/D SmolVLA-derived models."""
|
| 177 |
+
|
| 178 |
+
def __init__(self, model: Any, spec: AdapterSpec, **kwargs: Any) -> None:
|
| 179 |
+
super().__init__(model, spec, **kwargs)
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
class VITRAAdapter(ModelAdapter):
|
| 183 |
+
"""Adapter for VITRA or VITRA student action predictors."""
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
class TRexAdapter(ModelAdapter):
|
| 187 |
+
"""Adapter for the native T-Rex slow/fast predictor.
|
| 188 |
+
|
| 189 |
+
T-Rex does not expose the ``[B, 16, 192]`` hand-action contract used by
|
| 190 |
+
this repository. Its released inference server returns a denormalized
|
| 191 |
+
native chunk, normally ``[8, 62]`` or ``[16, 62]`` (31 dimensions per
|
| 192 |
+
arm: 9-D delta EEF pose + 22-D hand target). The Table 1 evaluator uses
|
| 193 |
+
a two-hand MANO action space, so converting that native output is a
|
| 194 |
+
task-specific operation and must be supplied through ``action_converter``.
|
| 195 |
+
|
| 196 |
+
The adapter nevertheless handles the complete interface around that
|
| 197 |
+
operation: unbatched server responses, the ``actions`` response key,
|
| 198 |
+
native-shape validation, optional raw-action normalization, and metadata
|
| 199 |
+
recording. If a predictor already emits the canonical action, no
|
| 200 |
+
converter is needed.
|
| 201 |
+
"""
|
| 202 |
+
|
| 203 |
+
NATIVE_ACTION_DIMS = (29, 31, 58, 62)
|
| 204 |
+
NATIVE_HORIZONS = (8, 16)
|
| 205 |
+
|
| 206 |
+
def __init__(self, model: Any, spec: AdapterSpec, **kwargs: Any) -> None:
|
| 207 |
+
native_converter = kwargs.pop("action_converter", None)
|
| 208 |
+
|
| 209 |
+
def convert(action: torch.Tensor) -> torch.Tensor:
|
| 210 |
+
# The ZMQ server returns a single chunk as [T, D]; model adapters
|
| 211 |
+
# use a batch dimension everywhere else.
|
| 212 |
+
if action.ndim == 2:
|
| 213 |
+
action = action.unsqueeze(0)
|
| 214 |
+
|
| 215 |
+
if action.ndim == 3 and tuple(action.shape[1:]) == (CHUNK_SIZE, ACTION_DIM):
|
| 216 |
+
return action
|
| 217 |
+
|
| 218 |
+
native_shape = tuple(action.shape[-2:]) if action.ndim == 3 else None
|
| 219 |
+
if native_shape is None or native_shape[0] not in self.NATIVE_HORIZONS or native_shape[1] not in self.NATIVE_ACTION_DIMS:
|
| 220 |
+
raise ValueError(
|
| 221 |
+
"T-Rex predictor must emit canonical [B,16,192] or a "
|
| 222 |
+
"native [T,D] / [B,T,D] chunk with T in "
|
| 223 |
+
f"{self.NATIVE_HORIZONS} and D in {self.NATIVE_ACTION_DIMS}; "
|
| 224 |
+
f"got {tuple(action.shape)}"
|
| 225 |
+
)
|
| 226 |
+
if native_converter is None:
|
| 227 |
+
raise ValueError(
|
| 228 |
+
"T-Rex emits native EEF/hand actions "
|
| 229 |
+
f"{native_shape}; provide action_converter to map them to "
|
| 230 |
+
"the canonical [B,16,192] MANO action space"
|
| 231 |
+
)
|
| 232 |
+
converted = native_converter(action)
|
| 233 |
+
if not isinstance(converted, torch.Tensor):
|
| 234 |
+
converted = torch.as_tensor(converted, device=action.device)
|
| 235 |
+
if converted.ndim == 2:
|
| 236 |
+
converted = converted.unsqueeze(0)
|
| 237 |
+
return converted
|
| 238 |
+
|
| 239 |
+
super().__init__(model, spec, action_converter=convert, **kwargs)
|
| 240 |
+
|
| 241 |
+
def __call__(
|
| 242 |
+
self,
|
| 243 |
+
observation: Any,
|
| 244 |
+
*,
|
| 245 |
+
action_mask: torch.Tensor | None = None,
|
| 246 |
+
edit_mask: torch.Tensor | None = None,
|
| 247 |
+
) -> BaseActionContext:
|
| 248 |
+
context = super().__call__(
|
| 249 |
+
observation, action_mask=action_mask, edit_mask=edit_mask
|
| 250 |
+
)
|
| 251 |
+
metadata = dict(context.metadata)
|
| 252 |
+
metadata.update(
|
| 253 |
+
{
|
| 254 |
+
"native_interface": "trex_slow_fast_cascaded_flow",
|
| 255 |
+
"native_action_dims": self.NATIVE_ACTION_DIMS,
|
| 256 |
+
"native_action_horizons": self.NATIVE_HORIZONS,
|
| 257 |
+
"has_external_memory_tokens": False,
|
| 258 |
+
"action_is_final": True,
|
| 259 |
+
}
|
| 260 |
+
)
|
| 261 |
+
# T-Rex's cached KV is an internal inference state, not a reusable
|
| 262 |
+
# memory-token tensor. Keeping it absent routes the model through the
|
| 263 |
+
# memory-free editor when a separate editor is explicitly requested.
|
| 264 |
+
return replace(context, memory_tokens=None, metadata=metadata)
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
class TWLAAdapter(ModelAdapter):
|
| 268 |
+
"""Adapter for the local T-WLA/V4-D checkpoint family.
|
| 269 |
+
|
| 270 |
+
The current T-WLA checkpoint exposes ``select_action_context`` and returns
|
| 271 |
+
the canonical full action together with ``final_memory``/``m_f``. This is
|
| 272 |
+
intentionally a named adapter so Table 1 configuration can distinguish
|
| 273 |
+
T-WLA from the MemoryVLA comparison row while sharing the same validated
|
| 274 |
+
contract.
|
| 275 |
+
"""
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
class MemoryVLAAdapter(SmolVLAAdapter):
|
| 279 |
+
"""Adapter configuration for the official MemoryVLA baseline.
|
| 280 |
+
|
| 281 |
+
Official MemoryVLA checkpoints emit robot-native end-effector actions and
|
| 282 |
+
keep cognitive/perceptual memories inside the policy. They do not emit
|
| 283 |
+
the canonical human-MANO action or reusable ``[B,N,D]`` memory tensor.
|
| 284 |
+
Therefore this adapter must be constructed with a predictor that includes
|
| 285 |
+
the task-specific human action head; it must not silently reuse the local
|
| 286 |
+
V4-D/T-WLA checkpoint.
|
| 287 |
+
"""
|
model/tachin_paired30_cache_20260923/code/src/tactile_editor/native_tachin.py
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Native 15-region / 880-cell Tachin tactile editors.
|
| 2 |
+
|
| 3 |
+
This module deliberately does not reshape the glove into the unrelated
|
| 4 |
+
EgoTouch 21x21 grid. One shared spatial encoder is applied to each native
|
| 5 |
+
sensor region, followed by sensor and temporal attention.
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from dataclasses import dataclass
|
| 10 |
+
import math
|
| 11 |
+
|
| 12 |
+
import torch
|
| 13 |
+
from torch import nn
|
| 14 |
+
|
| 15 |
+
from .action_layout import FULL_HAND_GROUPS
|
| 16 |
+
from .direct_residual import ActionResidualBasisTrunk
|
| 17 |
+
from .rgb_world_editor import FrozenRGBWorldResidual, RGBWorldResidualOutput
|
| 18 |
+
from .single_stage_world import (TactileWorldModel, gather_action_window,
|
| 19 |
+
tactile_flow_pair)
|
| 20 |
+
from .unified_world import DirectMemoryActionEditor
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
SENSOR_SHAPES = ((16, 10), (4, 8), (13, 8), (9, 8), (3, 8), (4, 8),
|
| 24 |
+
(9, 8), (3, 8), (4, 8), (9, 8), (3, 8), (4, 8),
|
| 25 |
+
(9, 8), (3, 8), (13, 8))
|
| 26 |
+
SENSOR_OFFSETS = (0, 160, 192, 296, 368, 392, 424, 496, 520, 552, 624,
|
| 27 |
+
648, 680, 752, 776, 880)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class NativeTachinEncoder(nn.Module):
|
| 31 |
+
"""Encode masked native cells into one 64-D token per hand."""
|
| 32 |
+
|
| 33 |
+
def __init__(self, pressure_mean: torch.Tensor, pressure_scale: torch.Tensor,
|
| 34 |
+
hidden_dim: int = 64) -> None:
|
| 35 |
+
super().__init__()
|
| 36 |
+
if tuple(pressure_mean.shape) != (2, 880) or tuple(pressure_scale.shape) != (2, 880):
|
| 37 |
+
raise ValueError("Tachin pressure statistics must be [2,880]")
|
| 38 |
+
self.register_buffer("pressure_mean", pressure_mean.float(), persistent=True)
|
| 39 |
+
self.register_buffer("pressure_scale", pressure_scale.float().clamp_min(1e-6), persistent=True)
|
| 40 |
+
self.spatial = nn.Sequential(
|
| 41 |
+
nn.Conv2d(2, 32, 3, padding=1), nn.GELU(),
|
| 42 |
+
nn.Conv2d(32, hidden_dim, 3, padding=1), nn.GELU(),
|
| 43 |
+
)
|
| 44 |
+
self.sensor_embedding = nn.Parameter(torch.randn(1, 15, hidden_dim) * 0.02)
|
| 45 |
+
sensor_layer = nn.TransformerEncoderLayer(hidden_dim, 4, hidden_dim * 4,
|
| 46 |
+
dropout=0.1, activation="gelu",
|
| 47 |
+
batch_first=True, norm_first=True)
|
| 48 |
+
self.sensor_blocks = nn.TransformerEncoder(sensor_layer, 1,
|
| 49 |
+
enable_nested_tensor=False)
|
| 50 |
+
self.temporal = nn.GRU(hidden_dim, hidden_dim, batch_first=True)
|
| 51 |
+
self.norm = nn.LayerNorm(hidden_dim)
|
| 52 |
+
|
| 53 |
+
def _encode(self, normalized: torch.Tensor, valid: torch.Tensor) -> torch.Tensor:
|
| 54 |
+
batch, history, hands = normalized.shape[:3]
|
| 55 |
+
sensor_tokens, sensor_valid = [], []
|
| 56 |
+
for sensor, ((height, width), start, stop) in enumerate(
|
| 57 |
+
zip(SENSOR_SHAPES, SENSOR_OFFSETS[:-1], SENSOR_OFFSETS[1:])):
|
| 58 |
+
values = normalized[..., start:stop].reshape(batch * history * hands, 1, height, width)
|
| 59 |
+
masks = valid[..., start:stop].reshape(batch * history * hands, 1, height, width)
|
| 60 |
+
features = self.spatial(torch.cat((values, masks.to(values.dtype)), 1))
|
| 61 |
+
denominator = masks.sum((2, 3)).clamp_min(1).to(features.dtype)
|
| 62 |
+
pooled = (features * masks.to(features.dtype)).sum((2, 3)) / denominator
|
| 63 |
+
sensor_tokens.append(pooled)
|
| 64 |
+
sensor_valid.append(masks.any((1, 2, 3)))
|
| 65 |
+
tokens = torch.stack(sensor_tokens, 1) + self.sensor_embedding
|
| 66 |
+
sensor_mask = torch.stack(sensor_valid, 1)
|
| 67 |
+
safe_mask = sensor_mask.clone()
|
| 68 |
+
safe_mask[~safe_mask.any(1), 0] = True
|
| 69 |
+
encoded = self.sensor_blocks(tokens, src_key_padding_mask=~safe_mask)
|
| 70 |
+
pooled = (encoded * safe_mask[..., None]).sum(1) / safe_mask.sum(1, keepdim=True).clamp_min(1)
|
| 71 |
+
pooled = pooled.reshape(batch, history, hands, -1).permute(0, 2, 1, 3)
|
| 72 |
+
sequence = pooled.reshape(batch * hands, history, -1)
|
| 73 |
+
self.temporal.flatten_parameters()
|
| 74 |
+
temporal = self.temporal(sequence)[0].reshape(batch, hands, history, -1)
|
| 75 |
+
return self.norm(temporal[:, :, -1])
|
| 76 |
+
|
| 77 |
+
def forward(self, pressure: torch.Tensor, cell_valid: torch.Tensor,
|
| 78 |
+
sample_valid: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
| 79 |
+
if tuple(pressure.shape[1:]) != (8, 2, 880):
|
| 80 |
+
raise ValueError("pressure must be [B,8,2,880]")
|
| 81 |
+
if cell_valid.shape != pressure.shape or sample_valid.shape != pressure.shape[:3]:
|
| 82 |
+
raise ValueError("Tachin tactile masks have the wrong shape")
|
| 83 |
+
normalized = (torch.nan_to_num(pressure.float()) - self.pressure_mean) / self.pressure_scale
|
| 84 |
+
valid = cell_valid.bool() & sample_valid.bool()[..., None]
|
| 85 |
+
normalized = normalized * valid.to(normalized.dtype)
|
| 86 |
+
# Center against an exact train-mean reference with the same missing-cell
|
| 87 |
+
# pattern. Zero tactile evidence therefore produces an exact zero token.
|
| 88 |
+
combined = self._encode(torch.cat((normalized, torch.zeros_like(normalized)), 0),
|
| 89 |
+
torch.cat((valid, valid), 0))
|
| 90 |
+
actual, reference = combined.chunk(2, 0)
|
| 91 |
+
hand_valid = sample_valid[:, -1].bool() & cell_valid[:, -1].any(-1)
|
| 92 |
+
output = (actual - reference) * hand_valid[..., None].to(actual.dtype)
|
| 93 |
+
return output, hand_valid
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def allowed_mask(edit_mask: torch.Tensor, hand_valid: torch.Tensor) -> torch.Tensor:
|
| 97 |
+
allowed = edit_mask.bool().clone()
|
| 98 |
+
allowed[..., 138:] = False
|
| 99 |
+
allowed[..., :69] &= hand_valid[:, 0, None, None]
|
| 100 |
+
allowed[..., 69:138] &= hand_valid[:, 1, None, None]
|
| 101 |
+
return allowed
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
class NativeResidualEditor(nn.Module):
|
| 105 |
+
"""The identical memory-free editor used for SmolVLA, VITRA, and T-Rex."""
|
| 106 |
+
|
| 107 |
+
def __init__(self, encoder: NativeTachinEncoder, residual_bases: int = 8) -> None:
|
| 108 |
+
super().__init__()
|
| 109 |
+
self.encoder = encoder
|
| 110 |
+
self.action = ActionResidualBasisTrunk(torch.zeros(192), torch.ones(192),
|
| 111 |
+
residual_bases=residual_bases)
|
| 112 |
+
self.coefficients = nn.Sequential(nn.LayerNorm(128), nn.Linear(128, residual_bases, bias=False))
|
| 113 |
+
self.residual_bases = residual_bases
|
| 114 |
+
|
| 115 |
+
def forward(self, base_action, edit_mask, pressure, cell_valid, sample_valid):
|
| 116 |
+
tactile, hand_valid = self.encoder(pressure, cell_valid, sample_valid)
|
| 117 |
+
coefficients = self.coefficients(tactile.flatten(1))
|
| 118 |
+
# LayerNorm has trainable affine parameters. Without an explicit
|
| 119 |
+
# centered-feature gate, its learned bias can create a global action
|
| 120 |
+
# correction even when pressure is exactly the train-mean reference.
|
| 121 |
+
# Match the EgoTouch editor contract: no tactile evidence means an
|
| 122 |
+
# exact frozen-base plan, including after training.
|
| 123 |
+
active = hand_valid.any(1, keepdim=True) & (
|
| 124 |
+
tactile.abs().sum((1, 2), keepdim=False)[:, None] > 0
|
| 125 |
+
)
|
| 126 |
+
coefficients = coefficients * active.to(coefficients.dtype)
|
| 127 |
+
bases = self.action(base_action, torch.zeros(len(base_action), device=base_action.device))
|
| 128 |
+
raw_delta = (coefficients[:, :, None, None] * bases).sum(1) / math.sqrt(self.residual_bases)
|
| 129 |
+
allowed = allowed_mask(edit_mask, hand_valid)
|
| 130 |
+
delta = raw_delta * allowed.to(raw_delta.dtype)
|
| 131 |
+
return base_action.detach() + delta, delta, allowed, tactile, hand_valid
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
@dataclass
|
| 135 |
+
class NativeW3Output:
|
| 136 |
+
final_action: torch.Tensor
|
| 137 |
+
delta_action: torch.Tensor
|
| 138 |
+
allowed_mask: torch.Tensor
|
| 139 |
+
tactile_features: torch.Tensor
|
| 140 |
+
hand_valid: torch.Tensor
|
| 141 |
+
tactile_velocity: torch.Tensor | None = None
|
| 142 |
+
tactile_target: torch.Tensor | None = None
|
| 143 |
+
tactile_valid: torch.Tensor | None = None
|
| 144 |
+
rgb: RGBWorldResidualOutput | None = None
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
class NativeTachinW3Editor(nn.Module):
|
| 148 |
+
"""Real W3: memory action editor plus RGB and tactile world objectives."""
|
| 149 |
+
|
| 150 |
+
def __init__(self, encoder: NativeTachinEncoder, pressure_mean: torch.Tensor,
|
| 151 |
+
pressure_scale: torch.Tensor) -> None:
|
| 152 |
+
super().__init__()
|
| 153 |
+
self.encoder = encoder
|
| 154 |
+
self.register_buffer("pressure_mean", pressure_mean.float(), persistent=True)
|
| 155 |
+
self.register_buffer("pressure_scale", pressure_scale.float().clamp_min(1e-6), persistent=True)
|
| 156 |
+
self.action = DirectMemoryActionEditor(torch.zeros(192), torch.ones(192),
|
| 157 |
+
tactile_dim=64, articulation_only=False)
|
| 158 |
+
self.tactile_world = TactileWorldModel(tactile_dim=64, cells=880)
|
| 159 |
+
self.rgb_world = FrozenRGBWorldResidual(tactile_dim=64, memory_rows=8,
|
| 160 |
+
condition_dim=2304, hidden_dim=256)
|
| 161 |
+
|
| 162 |
+
def forward(self, *, base_action, edit_mask, pressure, cell_valid, sample_valid,
|
| 163 |
+
final_memory, future_pressure=None, future_cell_valid=None,
|
| 164 |
+
tactile_tau=None, tactile_noise=None, rgb_world_head=None,
|
| 165 |
+
image_tokens=None, clean_latent=None, rgb_valid=None,
|
| 166 |
+
rgb_tau=None, rgb_noise=None, compute_auxiliary=True):
|
| 167 |
+
tactile, hand_valid = self.encoder(pressure, cell_valid, sample_valid)
|
| 168 |
+
final, delta, allowed = self.action(base_action, final_memory, tactile, hand_valid,
|
| 169 |
+
torch.zeros(len(base_action), device=base_action.device),
|
| 170 |
+
edit_mask)
|
| 171 |
+
tactile_velocity = tactile_target = tactile_valid = None
|
| 172 |
+
rgb = None
|
| 173 |
+
if compute_auxiliary:
|
| 174 |
+
required = (future_pressure, future_cell_valid, tactile_tau, tactile_noise,
|
| 175 |
+
rgb_world_head, image_tokens, clean_latent, rgb_valid, rgb_tau, rgb_noise)
|
| 176 |
+
if any(value is None for value in required):
|
| 177 |
+
raise ValueError("W3 auxiliary inputs are incomplete")
|
| 178 |
+
tactile_valid = future_cell_valid.bool()
|
| 179 |
+
clean = ((future_pressure.float() - self.pressure_mean) / self.pressure_scale)
|
| 180 |
+
clean = clean * tactile_valid.to(clean.dtype)
|
| 181 |
+
noisy, tactile_target = tactile_flow_pair(clean, tactile_valid, tactile_tau, tactile_noise)
|
| 182 |
+
action_window, action_valid = gather_action_window(final.detach(),
|
| 183 |
+
torch.zeros(len(final), device=final.device), steps=8)
|
| 184 |
+
tactile_velocity = self.tactile_world.velocity(noisy, tactile_tau,
|
| 185 |
+
final_memory=final_memory.detach(), tactile_features=tactile,
|
| 186 |
+
hand_valid=hand_valid, action_window=action_window, action_valid=action_valid)
|
| 187 |
+
rgb = self.rgb_world(world_head=rgb_world_head, image_tokens=image_tokens,
|
| 188 |
+
final_memory=final_memory.detach(), tactile_features=tactile,
|
| 189 |
+
hand_valid=hand_valid, clean_latent=clean_latent, world_valid=rgb_valid,
|
| 190 |
+
tau=rgb_tau, noise=rgb_noise)
|
| 191 |
+
return NativeW3Output(final, delta, allowed, tactile, hand_valid,
|
| 192 |
+
tactile_velocity, tactile_target, tactile_valid, rgb)
|
| 193 |
+
|
| 194 |
+
def optimizer_groups(self, action_lr=1e-4, encoder_lr=1e-5,
|
| 195 |
+
world_adapter_lr=1e-4, world_block_lr=1e-4):
|
| 196 |
+
return [
|
| 197 |
+
{"params": self.action.parameters(), "lr": action_lr, "name": "action"},
|
| 198 |
+
{"params": self.encoder.parameters(), "lr": encoder_lr, "name": "encoder"},
|
| 199 |
+
{"params": self.tactile_world.parameters(), "lr": world_block_lr, "name": "tactile_world"},
|
| 200 |
+
{"params": self.rgb_world.parameters(), "lr": world_adapter_lr, "name": "rgb_world"},
|
| 201 |
+
]
|
model/tachin_paired30_cache_20260923/code/src/tactile_editor/rgb_world_editor.py
ADDED
|
@@ -0,0 +1,419 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Frozen-RGB-world regularization for a Stage-2 tactile action editor.
|
| 2 |
+
|
| 3 |
+
The RGB world model is a teacher and is never stored inside this module. It predicts
|
| 4 |
+
rectified-flow velocity from the current image tokens and Method-D memory. A
|
| 5 |
+
reference-centred tactile history may perturb only the eight memory-condition rows;
|
| 6 |
+
the frozen SANA then turns that perturbation into a latent-velocity residual.
|
| 7 |
+
|
| 8 |
+
The action path is equally strict: action/memory features create residual bases and
|
| 9 |
+
per-hand tactile features create their coefficients. There is no action-only Stage-2
|
| 10 |
+
residual path, so reference tactile is an exact frozen Stage-1 identity.
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
from dataclasses import dataclass
|
| 16 |
+
import math
|
| 17 |
+
|
| 18 |
+
import torch
|
| 19 |
+
from torch import nn
|
| 20 |
+
|
| 21 |
+
from .action_objectives import per_sample_action_loss
|
| 22 |
+
from .action_layout import ARTICULATION_GROUPS, FULL_ACTION_GROUPS, FULL_HAND_GROUPS
|
| 23 |
+
from .direct_residual import DirectTactileTrunk
|
| 24 |
+
from .single_stage_world import MemoryConditionedActionBasis
|
| 25 |
+
from .training import ACTION_GROUPS
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def _check_shape(value: torch.Tensor, expected: tuple[int | None, ...], name: str) -> None:
|
| 29 |
+
if value.ndim != len(expected):
|
| 30 |
+
raise ValueError(f"{name} must have {len(expected)} dimensions")
|
| 31 |
+
for actual, wanted in zip(value.shape, expected):
|
| 32 |
+
if wanted is not None and actual != wanted:
|
| 33 |
+
raise ValueError(f"{name} must have shape {expected}, got {tuple(value.shape)}")
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class TactileMemoryConditionModulator(nn.Module):
|
| 37 |
+
"""Map two hand-local tactile features to the eight W0 memory rows.
|
| 38 |
+
|
| 39 |
+
Both projections are bias-free and the output layers are zero-initialized. Thus a
|
| 40 |
+
zero/reference tactile feature produces an exact zero condition delta, including at
|
| 41 |
+
initialization and after training.
|
| 42 |
+
"""
|
| 43 |
+
|
| 44 |
+
def __init__(self, tactile_dim: int = 128, memory_rows: int = 8,
|
| 45 |
+
condition_dim: int = 2304, hidden_dim: int = 256) -> None:
|
| 46 |
+
super().__init__()
|
| 47 |
+
self.memory_rows = memory_rows
|
| 48 |
+
self.condition_dim = condition_dim
|
| 49 |
+
self.input = nn.ModuleList(
|
| 50 |
+
nn.Linear(tactile_dim, hidden_dim, bias=False) for _ in range(2)
|
| 51 |
+
)
|
| 52 |
+
self.output = nn.ModuleList(
|
| 53 |
+
nn.Linear(hidden_dim, memory_rows * condition_dim, bias=False)
|
| 54 |
+
for _ in range(2)
|
| 55 |
+
)
|
| 56 |
+
for layer in self.output:
|
| 57 |
+
nn.init.zeros_(layer.weight)
|
| 58 |
+
|
| 59 |
+
def forward(self, tactile_features: torch.Tensor,
|
| 60 |
+
hand_valid: torch.Tensor) -> torch.Tensor:
|
| 61 |
+
_check_shape(tactile_features, (None, 2, None), "tactile_features")
|
| 62 |
+
if hand_valid.shape != tactile_features.shape[:2]:
|
| 63 |
+
raise ValueError("hand_valid must be [B,2]")
|
| 64 |
+
pieces = []
|
| 65 |
+
for hand in range(2):
|
| 66 |
+
feature = tactile_features[:, hand]
|
| 67 |
+
hidden = torch.nn.functional.gelu(self.input[hand](feature))
|
| 68 |
+
delta = self.output[hand](hidden).reshape(
|
| 69 |
+
len(feature), self.memory_rows, self.condition_dim
|
| 70 |
+
)
|
| 71 |
+
pieces.append(delta * hand_valid[:, hand, None, None].to(delta.dtype))
|
| 72 |
+
return pieces[0] + pieces[1]
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
@dataclass
|
| 76 |
+
class RGBWorldResidualOutput:
|
| 77 |
+
predicted_residual: torch.Tensor
|
| 78 |
+
target_residual: torch.Tensor
|
| 79 |
+
baseline_velocity: torch.Tensor
|
| 80 |
+
tactile_velocity: torch.Tensor
|
| 81 |
+
valid: torch.Tensor
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
class FrozenRGBWorldResidual(nn.Module):
|
| 85 |
+
"""Trainable tactile condition in front of a caller-owned, frozen RGB W0."""
|
| 86 |
+
|
| 87 |
+
def __init__(self, tactile_dim: int = 128, memory_rows: int = 8,
|
| 88 |
+
condition_dim: int = 2304, hidden_dim: int = 256) -> None:
|
| 89 |
+
super().__init__()
|
| 90 |
+
self.modulator = TactileMemoryConditionModulator(
|
| 91 |
+
tactile_dim=tactile_dim,
|
| 92 |
+
memory_rows=memory_rows,
|
| 93 |
+
condition_dim=condition_dim,
|
| 94 |
+
hidden_dim=hidden_dim,
|
| 95 |
+
)
|
| 96 |
+
|
| 97 |
+
def forward(
|
| 98 |
+
self,
|
| 99 |
+
*,
|
| 100 |
+
world_head,
|
| 101 |
+
image_tokens: torch.Tensor,
|
| 102 |
+
final_memory: torch.Tensor,
|
| 103 |
+
tactile_features: torch.Tensor,
|
| 104 |
+
hand_valid: torch.Tensor,
|
| 105 |
+
clean_latent: torch.Tensor,
|
| 106 |
+
world_valid: torch.Tensor,
|
| 107 |
+
tau: torch.Tensor,
|
| 108 |
+
noise: torch.Tensor,
|
| 109 |
+
) -> RGBWorldResidualOutput:
|
| 110 |
+
_check_shape(image_tokens, (None, None, None), "image_tokens")
|
| 111 |
+
_check_shape(final_memory, (len(image_tokens), 8, None), "final_memory")
|
| 112 |
+
_check_shape(clean_latent, (len(image_tokens), None, None, None, None), "clean_latent")
|
| 113 |
+
b, horizons = clean_latent.shape[:2]
|
| 114 |
+
if world_valid.shape != (b, horizons):
|
| 115 |
+
raise ValueError("world_valid must be [B,H]")
|
| 116 |
+
if tau.shape != (b, horizons):
|
| 117 |
+
raise ValueError("tau must be [B,H]")
|
| 118 |
+
if noise.shape != clean_latent.shape:
|
| 119 |
+
raise ValueError("noise must match clean_latent")
|
| 120 |
+
|
| 121 |
+
# Replace invalid values before arithmetic: NaN * 0 is still NaN.
|
| 122 |
+
valid5 = world_valid[..., None, None, None]
|
| 123 |
+
clean = torch.where(valid5, clean_latent.float(), torch.zeros_like(clean_latent.float()))
|
| 124 |
+
eps = torch.where(valid5, noise.float(), torch.zeros_like(noise.float()))
|
| 125 |
+
mixed = (1.0 - tau[..., None, None, None].float()) * clean
|
| 126 |
+
mixed = mixed + tau[..., None, None, None].float() * eps
|
| 127 |
+
target_velocity = eps - clean
|
| 128 |
+
|
| 129 |
+
adapter_dtype = next(world_head.adapter.parameters()).dtype
|
| 130 |
+
base_condition = world_head.build_condition(
|
| 131 |
+
image_tokens.to(adapter_dtype), final_memory.to(adapter_dtype)
|
| 132 |
+
)
|
| 133 |
+
memory_rows = self.modulator.memory_rows
|
| 134 |
+
if base_condition.shape[1] < memory_rows:
|
| 135 |
+
raise ValueError("world condition has fewer rows than the memory modulator")
|
| 136 |
+
delta = self.modulator(tactile_features, hand_valid).to(base_condition.dtype)
|
| 137 |
+
tactile_condition = base_condition.clone()
|
| 138 |
+
tactile_condition[:, -memory_rows:] = tactile_condition[:, -memory_rows:] + delta
|
| 139 |
+
|
| 140 |
+
adapter_horizons = getattr(world_head.adapter.cfg, "num_horizons", 1)
|
| 141 |
+
base_rows, tactile_rows = [], []
|
| 142 |
+
for horizon in range(horizons):
|
| 143 |
+
if adapter_horizons == horizons:
|
| 144 |
+
base_rows.append(world_head.adapter.for_horizon(base_condition, horizon))
|
| 145 |
+
tactile_rows.append(world_head.adapter.for_horizon(tactile_condition, horizon))
|
| 146 |
+
elif adapter_horizons == 1:
|
| 147 |
+
base_rows.append(base_condition)
|
| 148 |
+
tactile_rows.append(tactile_condition)
|
| 149 |
+
else:
|
| 150 |
+
raise ValueError(
|
| 151 |
+
"world adapter horizon count must be one or match clean_latent"
|
| 152 |
+
)
|
| 153 |
+
base_flat = torch.cat(base_rows, dim=0)
|
| 154 |
+
tactile_flat = torch.cat(tactile_rows, dim=0)
|
| 155 |
+
|
| 156 |
+
# Horizon-major flattening is required by W0's modulation horizon embedding.
|
| 157 |
+
flatten = lambda value: value.transpose(0, 1).reshape( # noqa: E731
|
| 158 |
+
horizons * b, *value.shape[2:]
|
| 159 |
+
)
|
| 160 |
+
mixed_flat = flatten(mixed).to(base_flat.dtype)
|
| 161 |
+
tau_flat = tau.transpose(0, 1).reshape(horizons * b).to(base_flat.dtype)
|
| 162 |
+
|
| 163 |
+
with torch.no_grad():
|
| 164 |
+
baseline = world_head.predict_velocity(mixed_flat, tau_flat, base_flat)
|
| 165 |
+
# W0 parameters are frozen, but this call keeps autograd with respect to the
|
| 166 |
+
# tactile-conditioned input.
|
| 167 |
+
tactile_velocity = world_head.predict_velocity(
|
| 168 |
+
mixed_flat, tau_flat, tactile_flat
|
| 169 |
+
)
|
| 170 |
+
target_flat = flatten(target_velocity)
|
| 171 |
+
predicted_residual = tactile_velocity.float() - baseline.float().detach()
|
| 172 |
+
target_residual = target_flat.float() - baseline.float().detach()
|
| 173 |
+
return RGBWorldResidualOutput(
|
| 174 |
+
predicted_residual=predicted_residual,
|
| 175 |
+
target_residual=target_residual.detach(),
|
| 176 |
+
baseline_velocity=baseline.detach(),
|
| 177 |
+
tactile_velocity=tactile_velocity,
|
| 178 |
+
valid=world_valid.transpose(0, 1).reshape(horizons * b),
|
| 179 |
+
)
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def roi_weighted_world_loss(output: RGBWorldResidualOutput,
|
| 183 |
+
roi: torch.Tensor,
|
| 184 |
+
*, background_weight: float = 0.1) -> torch.Tensor:
|
| 185 |
+
"""MSE on latent velocity with hand ROI weight 1 and background weight 0.1."""
|
| 186 |
+
if not 0 <= background_weight <= 1:
|
| 187 |
+
raise ValueError("background_weight must be in [0,1]")
|
| 188 |
+
if roi.ndim != 4:
|
| 189 |
+
raise ValueError("roi must be [B,H,Hlatent,Wlatent]")
|
| 190 |
+
flat_roi = roi.transpose(0, 1).reshape(-1, *roi.shape[2:]).bool()
|
| 191 |
+
if flat_roi.shape[0] != output.predicted_residual.shape[0]:
|
| 192 |
+
raise ValueError("roi rows must match world residual rows")
|
| 193 |
+
if flat_roi.shape[1:] != output.predicted_residual.shape[2:]:
|
| 194 |
+
raise ValueError("roi spatial shape must match world latent")
|
| 195 |
+
valid = output.valid.bool()
|
| 196 |
+
if not valid.any():
|
| 197 |
+
raise ValueError("the batch has no valid RGB world targets")
|
| 198 |
+
weight = torch.full_like(flat_roi, background_weight, dtype=torch.float32)
|
| 199 |
+
weight = torch.where(flat_roi, torch.ones_like(weight), weight)
|
| 200 |
+
error = (output.predicted_residual - output.target_residual).square()
|
| 201 |
+
weighted = error * weight[:, None].to(error.device)
|
| 202 |
+
per_row = weighted.flatten(1).sum(1) / (
|
| 203 |
+
weight.flatten(1).sum(1).to(error.device) * error.shape[1]
|
| 204 |
+
).clamp_min(1)
|
| 205 |
+
return per_row[valid].mean()
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
@dataclass
|
| 209 |
+
class Stage2Output:
|
| 210 |
+
final_action: torch.Tensor
|
| 211 |
+
delta_action: torch.Tensor
|
| 212 |
+
coefficients: torch.Tensor
|
| 213 |
+
tactile_features: torch.Tensor
|
| 214 |
+
tactile_history_features: torch.Tensor
|
| 215 |
+
hand_valid: torch.Tensor
|
| 216 |
+
allowed_mask: torch.Tensor
|
| 217 |
+
world: RGBWorldResidualOutput | None
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
class RGBWorldRegularizedTactileEditor(nn.Module):
|
| 221 |
+
"""Strict per-hand Stage-2 action editor with an RGB-world auxiliary branch."""
|
| 222 |
+
|
| 223 |
+
def __init__(
|
| 224 |
+
self,
|
| 225 |
+
*,
|
| 226 |
+
action_mean: torch.Tensor,
|
| 227 |
+
action_std: torch.Tensor,
|
| 228 |
+
pressure_masks: torch.Tensor,
|
| 229 |
+
reference_pressure: torch.Tensor,
|
| 230 |
+
residual_bases: int = 8,
|
| 231 |
+
articulation_only: bool = False,
|
| 232 |
+
memory_dim: int = 960,
|
| 233 |
+
hidden_dim: int = 256,
|
| 234 |
+
world_condition_dim: int = 2304,
|
| 235 |
+
) -> None:
|
| 236 |
+
super().__init__()
|
| 237 |
+
self.tactile = DirectTactileTrunk(
|
| 238 |
+
pressure_masks,
|
| 239 |
+
reference_pressure,
|
| 240 |
+
mode="history_delta",
|
| 241 |
+
coefficient_scope="hand_local",
|
| 242 |
+
residual_bases=residual_bases,
|
| 243 |
+
)
|
| 244 |
+
self.action = MemoryConditionedActionBasis(
|
| 245 |
+
action_mean,
|
| 246 |
+
action_std,
|
| 247 |
+
memory_dim=memory_dim,
|
| 248 |
+
hidden_dim=hidden_dim,
|
| 249 |
+
residual_bases=2 * residual_bases,
|
| 250 |
+
)
|
| 251 |
+
self.rgb_world = FrozenRGBWorldResidual(
|
| 252 |
+
tactile_dim=128,
|
| 253 |
+
memory_rows=8,
|
| 254 |
+
condition_dim=world_condition_dim,
|
| 255 |
+
hidden_dim=hidden_dim,
|
| 256 |
+
)
|
| 257 |
+
self.residual_bases = residual_bases
|
| 258 |
+
self.articulation_only = articulation_only
|
| 259 |
+
component = torch.zeros(1, 2, 1, 192, dtype=torch.bool)
|
| 260 |
+
groups = (
|
| 261 |
+
ARTICULATION_GROUPS if articulation_only else FULL_HAND_GROUPS
|
| 262 |
+
)
|
| 263 |
+
for hand, (start, stop) in enumerate(groups):
|
| 264 |
+
component[:, hand, :, start:stop] = True
|
| 265 |
+
self.register_buffer("hand_component_mask", component, persistent=True)
|
| 266 |
+
|
| 267 |
+
def forward(
|
| 268 |
+
self,
|
| 269 |
+
*,
|
| 270 |
+
anchor_action: torch.Tensor,
|
| 271 |
+
edit_mask: torch.Tensor,
|
| 272 |
+
pressure: torch.Tensor,
|
| 273 |
+
temporal_valid: torch.Tensor,
|
| 274 |
+
decision_seconds: torch.Tensor,
|
| 275 |
+
final_memory: torch.Tensor,
|
| 276 |
+
world_head=None,
|
| 277 |
+
image_tokens: torch.Tensor | None = None,
|
| 278 |
+
clean_latent: torch.Tensor | None = None,
|
| 279 |
+
world_valid: torch.Tensor | None = None,
|
| 280 |
+
tau: torch.Tensor | None = None,
|
| 281 |
+
noise: torch.Tensor | None = None,
|
| 282 |
+
) -> Stage2Output:
|
| 283 |
+
coefficients, hand_valid, tactile_features, tactile_history = self.tactile(
|
| 284 |
+
pressure, temporal_valid
|
| 285 |
+
)
|
| 286 |
+
all_bases = self.action(anchor_action, final_memory, decision_seconds)
|
| 287 |
+
bases = all_bases.reshape(
|
| 288 |
+
len(anchor_action), 2, self.residual_bases, 16, 192
|
| 289 |
+
)
|
| 290 |
+
per_hand = (coefficients[..., None, None] * bases).sum(2) / math.sqrt(
|
| 291 |
+
self.residual_bases
|
| 292 |
+
)
|
| 293 |
+
hand_allowed = edit_mask[:, None].bool() & self.hand_component_mask
|
| 294 |
+
hand_allowed = hand_allowed & hand_valid[:, :, None, None]
|
| 295 |
+
delta = (per_hand * hand_allowed.to(per_hand.dtype)).sum(1)
|
| 296 |
+
final_action = anchor_action.detach() + delta
|
| 297 |
+
|
| 298 |
+
supplied = (
|
| 299 |
+
world_head is not None,
|
| 300 |
+
image_tokens is not None,
|
| 301 |
+
clean_latent is not None,
|
| 302 |
+
world_valid is not None,
|
| 303 |
+
tau is not None,
|
| 304 |
+
noise is not None,
|
| 305 |
+
)
|
| 306 |
+
if any(supplied) and not all(supplied):
|
| 307 |
+
raise ValueError("all RGB world inputs must be supplied together")
|
| 308 |
+
world = None
|
| 309 |
+
if all(supplied):
|
| 310 |
+
world = self.rgb_world(
|
| 311 |
+
world_head=world_head,
|
| 312 |
+
image_tokens=image_tokens,
|
| 313 |
+
final_memory=final_memory.detach(),
|
| 314 |
+
tactile_features=tactile_features,
|
| 315 |
+
hand_valid=hand_valid,
|
| 316 |
+
clean_latent=clean_latent,
|
| 317 |
+
world_valid=world_valid,
|
| 318 |
+
tau=tau,
|
| 319 |
+
noise=noise,
|
| 320 |
+
)
|
| 321 |
+
return Stage2Output(
|
| 322 |
+
final_action=final_action,
|
| 323 |
+
delta_action=delta,
|
| 324 |
+
coefficients=coefficients,
|
| 325 |
+
tactile_features=tactile_features,
|
| 326 |
+
tactile_history_features=tactile_history,
|
| 327 |
+
hand_valid=hand_valid,
|
| 328 |
+
allowed_mask=hand_allowed.any(1),
|
| 329 |
+
world=world,
|
| 330 |
+
)
|
| 331 |
+
|
| 332 |
+
def objective(
|
| 333 |
+
self,
|
| 334 |
+
output: Stage2Output,
|
| 335 |
+
target_action: torch.Tensor,
|
| 336 |
+
edit_mask: torch.Tensor,
|
| 337 |
+
*,
|
| 338 |
+
roi: torch.Tensor | None,
|
| 339 |
+
lambda_world: float,
|
| 340 |
+
lambda_delta: float = 1e-4,
|
| 341 |
+
lambda_coefficient: float = 1e-5,
|
| 342 |
+
background_weight: float = 0.1,
|
| 343 |
+
) -> tuple[torch.Tensor, dict[str, torch.Tensor]]:
|
| 344 |
+
if min(lambda_world, lambda_delta, lambda_coefficient) < 0:
|
| 345 |
+
raise ValueError("loss weights must be non-negative")
|
| 346 |
+
mask = edit_mask.bool() & output.allowed_mask
|
| 347 |
+
present = mask.any((1, 2))
|
| 348 |
+
if not present.any():
|
| 349 |
+
raise ValueError("the batch has no tactile-editable action target")
|
| 350 |
+
groups = ARTICULATION_GROUPS if self.articulation_only else FULL_ACTION_GROUPS
|
| 351 |
+
action = per_sample_action_loss(
|
| 352 |
+
output.final_action[present], target_action[present], mask[present], groups=groups
|
| 353 |
+
).mean()
|
| 354 |
+
delta_l2 = output.delta_action.float().square().masked_fill(~mask, 0).sum()
|
| 355 |
+
delta_l2 = delta_l2 / mask.sum().clamp_min(1)
|
| 356 |
+
coefficient_l2 = output.coefficients.float().square().mean()
|
| 357 |
+
if lambda_world:
|
| 358 |
+
if output.world is None or roi is None:
|
| 359 |
+
raise ValueError("RGB world output and roi are required")
|
| 360 |
+
world = roi_weighted_world_loss(
|
| 361 |
+
output.world, roi, background_weight=background_weight
|
| 362 |
+
)
|
| 363 |
+
else:
|
| 364 |
+
world = action.new_zeros(())
|
| 365 |
+
total = action + lambda_world * world
|
| 366 |
+
total = total + lambda_delta * delta_l2 + lambda_coefficient * coefficient_l2
|
| 367 |
+
return total, {
|
| 368 |
+
"loss": total.detach(),
|
| 369 |
+
"action_loss": action.detach(),
|
| 370 |
+
"world_loss": world.detach(),
|
| 371 |
+
"delta_l2": delta_l2.detach(),
|
| 372 |
+
"coefficient_l2": coefficient_l2.detach(),
|
| 373 |
+
}
|
| 374 |
+
|
| 375 |
+
def optimizer_groups(
|
| 376 |
+
self,
|
| 377 |
+
*,
|
| 378 |
+
action_lr: float = 1e-4,
|
| 379 |
+
tactile_gru_lr: float = 1e-5,
|
| 380 |
+
tactile_cnn_lr: float = 3e-6,
|
| 381 |
+
world_modulator_lr: float = 1e-4,
|
| 382 |
+
) -> list[dict]:
|
| 383 |
+
groups = [
|
| 384 |
+
{
|
| 385 |
+
"name": "action",
|
| 386 |
+
"params": [
|
| 387 |
+
*self.action.parameters(),
|
| 388 |
+
*self.tactile.coefficient_head.parameters(),
|
| 389 |
+
],
|
| 390 |
+
"lr": action_lr,
|
| 391 |
+
},
|
| 392 |
+
{
|
| 393 |
+
"name": "tactile.cnn",
|
| 394 |
+
"params": list(self.tactile.frame_encoder.parameters()),
|
| 395 |
+
"lr": tactile_cnn_lr,
|
| 396 |
+
},
|
| 397 |
+
{
|
| 398 |
+
"name": "world.modulator",
|
| 399 |
+
"params": list(self.rgb_world.parameters()),
|
| 400 |
+
"lr": world_modulator_lr,
|
| 401 |
+
},
|
| 402 |
+
]
|
| 403 |
+
if self.tactile.temporal is not None:
|
| 404 |
+
groups.append(
|
| 405 |
+
{
|
| 406 |
+
"name": "tactile.gru",
|
| 407 |
+
"params": list(self.tactile.temporal.parameters()),
|
| 408 |
+
"lr": tactile_gru_lr,
|
| 409 |
+
}
|
| 410 |
+
)
|
| 411 |
+
return groups
|
| 412 |
+
|
| 413 |
+
|
| 414 |
+
def ramp_world_weight(step: int, maximum: float, warmup_steps: int) -> float:
|
| 415 |
+
if maximum < 0 or warmup_steps < 0:
|
| 416 |
+
raise ValueError("maximum and warmup_steps must be non-negative")
|
| 417 |
+
if warmup_steps == 0:
|
| 418 |
+
return float(maximum)
|
| 419 |
+
return float(maximum) * min(1.0, max(0.0, step / warmup_steps))
|
model/tachin_paired30_cache_20260923/code/src/tactile_editor/runtime.py
ADDED
|
@@ -0,0 +1,376 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Timestamp-driven tactile buffers and pipelined VLA action control."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import bisect
|
| 6 |
+
import math
|
| 7 |
+
from dataclasses import dataclass, field
|
| 8 |
+
from typing import Callable
|
| 9 |
+
|
| 10 |
+
import torch
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class TactileRingBuffer:
|
| 14 |
+
"""Causally resample a native-rate stream to the encoder's 30 Hz contract."""
|
| 15 |
+
|
| 16 |
+
def __init__(self, history_frames: int = 8, model_hz: float = 30.0) -> None:
|
| 17 |
+
if history_frames <= 0 or model_hz <= 0:
|
| 18 |
+
raise ValueError("history_frames and model_hz must be positive")
|
| 19 |
+
self.history_frames = int(history_frames)
|
| 20 |
+
self.model_hz = float(model_hz)
|
| 21 |
+
self._timestamps: list[float] = []
|
| 22 |
+
self._pressure: list[torch.Tensor] = []
|
| 23 |
+
self._valid: list[torch.Tensor] = []
|
| 24 |
+
self.episode_id: int | str | None = None
|
| 25 |
+
|
| 26 |
+
@property
|
| 27 |
+
def sample_count(self) -> int:
|
| 28 |
+
return len(self._timestamps)
|
| 29 |
+
|
| 30 |
+
def reset(self, episode_id: int | str | None = None) -> None:
|
| 31 |
+
self._timestamps.clear()
|
| 32 |
+
self._pressure.clear()
|
| 33 |
+
self._valid.clear()
|
| 34 |
+
self.episode_id = episode_id
|
| 35 |
+
|
| 36 |
+
def append(
|
| 37 |
+
self,
|
| 38 |
+
timestamp: float,
|
| 39 |
+
pressure: torch.Tensor,
|
| 40 |
+
valid: torch.Tensor | None = None,
|
| 41 |
+
*,
|
| 42 |
+
episode_id: int | str | None = None,
|
| 43 |
+
) -> None:
|
| 44 |
+
if not math.isfinite(timestamp):
|
| 45 |
+
raise ValueError("tactile timestamp must be finite")
|
| 46 |
+
if tuple(pressure.shape) != (2, 21, 21):
|
| 47 |
+
raise ValueError("one tactile frame must be [2,21,21]")
|
| 48 |
+
if self.episode_id is not None and episode_id != self.episode_id:
|
| 49 |
+
self.reset(episode_id)
|
| 50 |
+
elif self.episode_id is None:
|
| 51 |
+
self.episode_id = episode_id
|
| 52 |
+
if self._timestamps and timestamp <= self._timestamps[-1]:
|
| 53 |
+
raise ValueError("tactile timestamps must be strictly increasing")
|
| 54 |
+
if valid is None:
|
| 55 |
+
valid = torch.ones(2, dtype=torch.bool, device=pressure.device)
|
| 56 |
+
if tuple(valid.shape) != (2,):
|
| 57 |
+
raise ValueError("tactile validity must be [2]")
|
| 58 |
+
self._timestamps.append(float(timestamp))
|
| 59 |
+
self._pressure.append(pressure.detach().clone())
|
| 60 |
+
self._valid.append(
|
| 61 |
+
valid.to(device=pressure.device, dtype=torch.bool).detach().clone()
|
| 62 |
+
)
|
| 63 |
+
# Retain one sample before the useful horizon. That sample is needed for
|
| 64 |
+
# causal zero-order hold when a target time falls between sensor frames.
|
| 65 |
+
horizon = (self.history_frames + 1) / self.model_hz
|
| 66 |
+
first_useful = bisect.bisect_left(self._timestamps, float(timestamp) - horizon)
|
| 67 |
+
remove_count = max(0, first_useful - 1)
|
| 68 |
+
if remove_count:
|
| 69 |
+
del self._timestamps[:remove_count]
|
| 70 |
+
del self._pressure[:remove_count]
|
| 71 |
+
del self._valid[:remove_count]
|
| 72 |
+
|
| 73 |
+
def window(self, timestamp: float) -> tuple[torch.Tensor, torch.Tensor]:
|
| 74 |
+
"""Return [1,8,2,21,21] and [1,8,2], using no future sample."""
|
| 75 |
+
|
| 76 |
+
if not self._timestamps:
|
| 77 |
+
raise RuntimeError("cannot build a tactile window from an empty buffer")
|
| 78 |
+
frames: list[torch.Tensor] = []
|
| 79 |
+
valids: list[torch.Tensor] = []
|
| 80 |
+
for offset in range(self.history_frames - 1, -1, -1):
|
| 81 |
+
target = float(timestamp) - offset / self.model_hz
|
| 82 |
+
index = bisect.bisect_right(self._timestamps, target + 1e-9) - 1
|
| 83 |
+
if index < 0:
|
| 84 |
+
# Repeat the earliest value for a stable tensor, but mark it padded.
|
| 85 |
+
frames.append(self._pressure[0])
|
| 86 |
+
valids.append(torch.zeros_like(self._valid[0]))
|
| 87 |
+
else:
|
| 88 |
+
frames.append(self._pressure[index])
|
| 89 |
+
valids.append(self._valid[index])
|
| 90 |
+
return torch.stack(frames)[None], torch.stack(valids)[None]
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
@dataclass
|
| 94 |
+
class ActionPlan:
|
| 95 |
+
"""A revision-safe full chunk. Committed actions are immutable."""
|
| 96 |
+
|
| 97 |
+
actions: torch.Tensor # [T,A], normally normalized inside the editor
|
| 98 |
+
started_at: float
|
| 99 |
+
action_hz: float
|
| 100 |
+
plan_id: int
|
| 101 |
+
action_mask: torch.Tensor | None = None # [T,A], valid Method-D action slots
|
| 102 |
+
_base_actions: torch.Tensor = field(init=False, repr=False)
|
| 103 |
+
|
| 104 |
+
def __post_init__(self) -> None:
|
| 105 |
+
if self.actions.ndim != 2:
|
| 106 |
+
raise ValueError("actions must be [T,A]")
|
| 107 |
+
if self.action_hz <= 0:
|
| 108 |
+
raise ValueError("action_hz must be positive")
|
| 109 |
+
self.actions = self.actions.detach().clone()
|
| 110 |
+
self._base_actions = self.actions.clone()
|
| 111 |
+
if self.action_mask is not None:
|
| 112 |
+
if self.action_mask.shape != self.actions.shape:
|
| 113 |
+
raise ValueError("action_mask and actions must have the same shape")
|
| 114 |
+
self.action_mask = (
|
| 115 |
+
self.action_mask.to(device=self.actions.device, dtype=torch.bool)
|
| 116 |
+
.detach()
|
| 117 |
+
.clone()
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
+
@property
|
| 121 |
+
def base_actions(self) -> torch.Tensor:
|
| 122 |
+
"""Original installed VLA output; callers cannot mutate the snapshot."""
|
| 123 |
+
return self._base_actions.clone()
|
| 124 |
+
|
| 125 |
+
def committed_steps(self, now: float, compute_latency_seconds: float = 0.0) -> int:
|
| 126 |
+
elapsed = max(0.0, float(now) - self.started_at + compute_latency_seconds)
|
| 127 |
+
# Subtract a small tolerance so exact action boundaries stay exact after
|
| 128 |
+
# floating-point subtraction (1.05 - 1.0 can be greater than 0.05).
|
| 129 |
+
return min(
|
| 130 |
+
len(self.actions), max(0, math.ceil(elapsed * self.action_hz - 1e-9))
|
| 131 |
+
)
|
| 132 |
+
|
| 133 |
+
def editable_mask(
|
| 134 |
+
self, now: float, compute_latency_seconds: float = 0.0
|
| 135 |
+
) -> torch.Tensor:
|
| 136 |
+
start = self.committed_steps(now, compute_latency_seconds)
|
| 137 |
+
result = torch.zeros_like(self.actions, dtype=torch.bool)
|
| 138 |
+
result[start:] = True
|
| 139 |
+
if self.action_mask is not None:
|
| 140 |
+
result &= self.action_mask
|
| 141 |
+
return result
|
| 142 |
+
|
| 143 |
+
def apply_revision(
|
| 144 |
+
self,
|
| 145 |
+
candidate: torch.Tensor,
|
| 146 |
+
*,
|
| 147 |
+
now: float,
|
| 148 |
+
compute_latency_seconds: float = 0.0,
|
| 149 |
+
expected_plan_id: int | None = None,
|
| 150 |
+
) -> torch.Tensor:
|
| 151 |
+
if candidate.shape != self.actions.shape:
|
| 152 |
+
raise ValueError("candidate and plan action shapes differ")
|
| 153 |
+
if expected_plan_id is not None and expected_plan_id != self.plan_id:
|
| 154 |
+
raise RuntimeError("stale tactile result targets an obsolete action plan")
|
| 155 |
+
mask = self.editable_mask(now, compute_latency_seconds)
|
| 156 |
+
self.actions = torch.where(mask, candidate, self.actions)
|
| 157 |
+
return self.actions
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
RevisionFn = Callable[
|
| 161 |
+
[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], torch.Tensor
|
| 162 |
+
]
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
def next_action_boundary(timestamp: float, action_hz: float) -> float:
|
| 166 |
+
"""Return the first 80 Hz-style action boundary at or after a timestamp."""
|
| 167 |
+
|
| 168 |
+
if action_hz <= 0:
|
| 169 |
+
raise ValueError("action_hz must be positive")
|
| 170 |
+
if not math.isfinite(timestamp):
|
| 171 |
+
raise ValueError("timestamp must be finite")
|
| 172 |
+
tick = math.ceil(float(timestamp) * action_hz - 1e-9)
|
| 173 |
+
return tick / action_hz
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
@dataclass(frozen=True)
|
| 177 |
+
class InferenceRequest:
|
| 178 |
+
"""One VLA request with its camera capture timestamp."""
|
| 179 |
+
|
| 180 |
+
request_id: int
|
| 181 |
+
captured_at: float
|
| 182 |
+
episode_id: int | str
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
@dataclass(frozen=True)
|
| 186 |
+
class ScheduledActionPlan:
|
| 187 |
+
"""A completed plan waiting for an atomic action-boundary install."""
|
| 188 |
+
|
| 189 |
+
request_id: int
|
| 190 |
+
plan: ActionPlan
|
| 191 |
+
ready_at: float
|
| 192 |
+
install_at: float
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
class PipelinedController:
|
| 196 |
+
"""Overlap VLA inference with execution and install each new action queue."""
|
| 197 |
+
|
| 198 |
+
def __init__(
|
| 199 |
+
self,
|
| 200 |
+
*,
|
| 201 |
+
control_hz: float = 10.0,
|
| 202 |
+
action_hz: float = 80.0,
|
| 203 |
+
tactile_model_hz: float = 30.0,
|
| 204 |
+
history_frames: int = 8,
|
| 205 |
+
compute_latency_seconds: float = 0.0,
|
| 206 |
+
) -> None:
|
| 207 |
+
if control_hz <= 0 or action_hz <= 0 or tactile_model_hz <= 0:
|
| 208 |
+
raise ValueError("runtime rates must be positive")
|
| 209 |
+
if history_frames <= 0:
|
| 210 |
+
raise ValueError("history_frames must be positive")
|
| 211 |
+
if compute_latency_seconds < 0:
|
| 212 |
+
raise ValueError("compute_latency_seconds must be non-negative")
|
| 213 |
+
self.control_hz = float(control_hz)
|
| 214 |
+
self.action_hz = float(action_hz)
|
| 215 |
+
self.compute_latency_seconds = float(compute_latency_seconds)
|
| 216 |
+
self.touch = TactileRingBuffer(history_frames, tactile_model_hz)
|
| 217 |
+
self.plan: ActionPlan | None = None
|
| 218 |
+
self.active_schedule: ScheduledActionPlan | None = None
|
| 219 |
+
self.pending_schedule: ScheduledActionPlan | None = None
|
| 220 |
+
self.inflight_request: InferenceRequest | None = None
|
| 221 |
+
self._episode_id: int | str | None = None
|
| 222 |
+
self._last_capture_at: float | None = None
|
| 223 |
+
self._next_request_id = 0
|
| 224 |
+
self._next_plan_id = 0
|
| 225 |
+
|
| 226 |
+
def reset(self, episode_id: int | str | None = None) -> None:
|
| 227 |
+
"""Clear every temporal state at an episode boundary."""
|
| 228 |
+
|
| 229 |
+
self.touch.reset(episode_id)
|
| 230 |
+
self.plan = None
|
| 231 |
+
self.active_schedule = None
|
| 232 |
+
self.pending_schedule = None
|
| 233 |
+
self.inflight_request = None
|
| 234 |
+
self._episode_id = episode_id
|
| 235 |
+
self._last_capture_at = None
|
| 236 |
+
|
| 237 |
+
def _activate_ready_plan(self, timestamp: float) -> None:
|
| 238 |
+
pending = self.pending_schedule
|
| 239 |
+
if pending is not None and float(timestamp) + 1e-9 >= pending.install_at:
|
| 240 |
+
self.active_schedule = pending
|
| 241 |
+
self.plan = pending.plan
|
| 242 |
+
self.pending_schedule = None
|
| 243 |
+
|
| 244 |
+
def begin_inference(
|
| 245 |
+
self, *, captured_at: float, episode_id: int | str
|
| 246 |
+
) -> InferenceRequest:
|
| 247 |
+
"""Start a VLA request while the current action plan remains active."""
|
| 248 |
+
|
| 249 |
+
if not math.isfinite(captured_at):
|
| 250 |
+
raise ValueError("capture timestamp must be finite")
|
| 251 |
+
if self._episode_id is not None and episode_id != self._episode_id:
|
| 252 |
+
self.reset(episode_id)
|
| 253 |
+
elif self._episode_id is None:
|
| 254 |
+
self.reset(episode_id)
|
| 255 |
+
self._activate_ready_plan(captured_at)
|
| 256 |
+
if self.inflight_request is not None:
|
| 257 |
+
raise RuntimeError("one VLA inference request is already in flight")
|
| 258 |
+
if self._last_capture_at is not None:
|
| 259 |
+
minimum = self._last_capture_at + 1.0 / self.control_hz
|
| 260 |
+
if captured_at < minimum - 1e-9:
|
| 261 |
+
raise RuntimeError(
|
| 262 |
+
"VLA capture requests exceed the configured control rate"
|
| 263 |
+
)
|
| 264 |
+
self._next_request_id += 1
|
| 265 |
+
request = InferenceRequest(
|
| 266 |
+
self._next_request_id, float(captured_at), episode_id
|
| 267 |
+
)
|
| 268 |
+
self.inflight_request = request
|
| 269 |
+
self._last_capture_at = float(captured_at)
|
| 270 |
+
return request
|
| 271 |
+
|
| 272 |
+
def complete_inference(
|
| 273 |
+
self,
|
| 274 |
+
full_action: torch.Tensor,
|
| 275 |
+
*,
|
| 276 |
+
request_id: int,
|
| 277 |
+
ready_at: float,
|
| 278 |
+
episode_id: int | str,
|
| 279 |
+
action_mask: torch.Tensor | None = None,
|
| 280 |
+
) -> ScheduledActionPlan:
|
| 281 |
+
"""Schedule an unedited VLA chunk for the next action boundary.
|
| 282 |
+
|
| 283 |
+
Pass the frozen backbone output, not a previously revised V1 output.
|
| 284 |
+
Subsequent touch updates always predict residuals from this snapshot.
|
| 285 |
+
"""
|
| 286 |
+
|
| 287 |
+
request = self.inflight_request
|
| 288 |
+
if request is None or request.request_id != request_id:
|
| 289 |
+
raise RuntimeError("stale or unknown VLA inference result")
|
| 290 |
+
if request.episode_id != episode_id or self._episode_id != episode_id:
|
| 291 |
+
raise RuntimeError("VLA inference result belongs to an obsolete episode")
|
| 292 |
+
if not math.isfinite(ready_at) or ready_at < request.captured_at:
|
| 293 |
+
raise ValueError("ready_at must be finite and no earlier than captured_at")
|
| 294 |
+
self._activate_ready_plan(ready_at)
|
| 295 |
+
if self.pending_schedule is not None:
|
| 296 |
+
raise RuntimeError(
|
| 297 |
+
"the previous VLA plan has not reached its install boundary"
|
| 298 |
+
)
|
| 299 |
+
install_at = next_action_boundary(ready_at, self.action_hz)
|
| 300 |
+
self._next_plan_id += 1
|
| 301 |
+
plan = ActionPlan(
|
| 302 |
+
full_action.detach().clone(),
|
| 303 |
+
started_at=install_at,
|
| 304 |
+
action_hz=self.action_hz,
|
| 305 |
+
plan_id=self._next_plan_id,
|
| 306 |
+
action_mask=action_mask,
|
| 307 |
+
)
|
| 308 |
+
self.inflight_request = None
|
| 309 |
+
scheduled = ScheduledActionPlan(
|
| 310 |
+
request_id=request_id,
|
| 311 |
+
plan=plan,
|
| 312 |
+
ready_at=float(ready_at),
|
| 313 |
+
install_at=install_at,
|
| 314 |
+
)
|
| 315 |
+
self.pending_schedule = scheduled
|
| 316 |
+
self._activate_ready_plan(ready_at)
|
| 317 |
+
return scheduled
|
| 318 |
+
|
| 319 |
+
def update_touch(
|
| 320 |
+
self,
|
| 321 |
+
pressure: torch.Tensor,
|
| 322 |
+
*,
|
| 323 |
+
timestamp: float,
|
| 324 |
+
episode_id: int | str,
|
| 325 |
+
valid: torch.Tensor | None = None,
|
| 326 |
+
revise: RevisionFn | None = None,
|
| 327 |
+
) -> ActionPlan | None:
|
| 328 |
+
"""Append one sensor frame and optionally operate a fast V1 revision."""
|
| 329 |
+
|
| 330 |
+
if self._episode_id is not None and episode_id != self._episode_id:
|
| 331 |
+
self.reset(episode_id)
|
| 332 |
+
elif self._episode_id is None:
|
| 333 |
+
self.reset(episode_id)
|
| 334 |
+
self.touch.append(timestamp, pressure, valid, episode_id=episode_id)
|
| 335 |
+
self._activate_ready_plan(timestamp)
|
| 336 |
+
if self.plan is None or revise is None:
|
| 337 |
+
return self.plan
|
| 338 |
+
# Capture the exact plan targeted by this callback. A new VLA result may
|
| 339 |
+
# replace ``self.plan`` while a slow tactile revision is still running.
|
| 340 |
+
target_plan = self.plan
|
| 341 |
+
expected_plan_id = target_plan.plan_id
|
| 342 |
+
window, window_valid = self.touch.window(timestamp)
|
| 343 |
+
edit_mask = target_plan.editable_mask(timestamp, self.compute_latency_seconds)
|
| 344 |
+
candidate = revise(
|
| 345 |
+
target_plan.base_actions[None], edit_mask[None], window, window_valid
|
| 346 |
+
)
|
| 347 |
+
if tuple(candidate.shape) != (1, *target_plan.actions.shape):
|
| 348 |
+
raise ValueError("revision callback must return [1,T,A]")
|
| 349 |
+
if self.plan is not target_plan or self.plan.plan_id != expected_plan_id:
|
| 350 |
+
raise RuntimeError("stale tactile result targets an obsolete action plan")
|
| 351 |
+
target_plan.apply_revision(
|
| 352 |
+
candidate[0],
|
| 353 |
+
now=timestamp,
|
| 354 |
+
compute_latency_seconds=self.compute_latency_seconds,
|
| 355 |
+
expected_plan_id=expected_plan_id,
|
| 356 |
+
)
|
| 357 |
+
return target_plan
|
| 358 |
+
|
| 359 |
+
def next_action(self, timestamp: float) -> torch.Tensor:
|
| 360 |
+
"""Return the next action relative to the plan installation time."""
|
| 361 |
+
|
| 362 |
+
self._activate_ready_plan(timestamp)
|
| 363 |
+
if self.plan is None:
|
| 364 |
+
raise RuntimeError(
|
| 365 |
+
"no active VLA action plan is available; use a safe hold"
|
| 366 |
+
)
|
| 367 |
+
index = math.floor(
|
| 368 |
+
(float(timestamp) - self.plan.started_at) * self.action_hz + 1e-9
|
| 369 |
+
)
|
| 370 |
+
if index < 0:
|
| 371 |
+
raise RuntimeError("the active VLA plan starts in the future")
|
| 372 |
+
if index >= len(self.plan.actions):
|
| 373 |
+
raise RuntimeError(
|
| 374 |
+
"the full action-chunk latency buffer is exhausted; use a safe hold"
|
| 375 |
+
)
|
| 376 |
+
return self.plan.actions[index]
|
model/tachin_paired30_cache_20260923/code/src/tactile_editor/touch_study.py
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Experimental incremental editors. These do not change the deployment defaults.
|
| 2 |
+
|
| 3 |
+
The immutable anchor is a frozen, calibrated VLA plan. All methods use the same
|
| 4 |
+
uncapped head and output mask. Modality dropout only affects conditioning.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from dataclasses import dataclass
|
| 8 |
+
import hashlib
|
| 9 |
+
import json
|
| 10 |
+
import math
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
from torch import nn
|
| 15 |
+
from torch.nn import functional as F
|
| 16 |
+
|
| 17 |
+
from .fusion import GatedCrossAttention
|
| 18 |
+
from .training import ACTION_GROUPS
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@dataclass(frozen=True)
|
| 22 |
+
class StudySetting:
|
| 23 |
+
name: str
|
| 24 |
+
encoder_lr: float = 0.0
|
| 25 |
+
constant: bool = False
|
| 26 |
+
condition_dropout: float = 0.0
|
| 27 |
+
touch_dropout: float = 0.0
|
| 28 |
+
dropout: float = 0.1
|
| 29 |
+
teacher_weight: float = 0.0
|
| 30 |
+
dynamics_weight: float = 0.0
|
| 31 |
+
innovation: bool = False
|
| 32 |
+
event_sampling: bool = False
|
| 33 |
+
gradient_balance: bool = False
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
SETTINGS = {
|
| 37 |
+
s.name: s
|
| 38 |
+
for s in (
|
| 39 |
+
StudySetting("constant", constant=True),
|
| 40 |
+
StudySetting("frozen"),
|
| 41 |
+
StudySetting("finetune", encoder_lr=1e-5),
|
| 42 |
+
StudySetting("ordinary_dropout", encoder_lr=1e-5, dropout=0.3),
|
| 43 |
+
StudySetting("condition_dropout", encoder_lr=1e-5, condition_dropout=0.2),
|
| 44 |
+
StudySetting("touch_dropout", encoder_lr=1e-5, touch_dropout=0.2),
|
| 45 |
+
StudySetting("teacher", encoder_lr=1e-5, teacher_weight=1.0),
|
| 46 |
+
StudySetting("dynamics", encoder_lr=1e-5, dynamics_weight=0.1),
|
| 47 |
+
StudySetting("innovation", innovation=True),
|
| 48 |
+
StudySetting("event_sampling", event_sampling=True),
|
| 49 |
+
StudySetting("gradient_balance", encoder_lr=1e-5, gradient_balance=True),
|
| 50 |
+
)
|
| 51 |
+
}
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def digest(path):
|
| 55 |
+
"""Return a streaming SHA-256 digest for provenance records."""
|
| 56 |
+
value = hashlib.sha256()
|
| 57 |
+
with Path(path).open("rb") as source:
|
| 58 |
+
for block in iter(lambda: source.read(8 * 1024 * 1024), b""):
|
| 59 |
+
value.update(block)
|
| 60 |
+
return value.hexdigest()
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def write_json(path, value):
|
| 64 |
+
"""Atomically replace a JSON result file."""
|
| 65 |
+
path = Path(path)
|
| 66 |
+
temporary = path.with_suffix(path.suffix + ".tmp")
|
| 67 |
+
temporary.write_text(json.dumps(value, indent=2, default=str) + "\n")
|
| 68 |
+
temporary.replace(path)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def learning_rate_factor(step, horizon, warmup):
|
| 72 |
+
"""Linear warmup followed by cosine decay to ten percent."""
|
| 73 |
+
if horizon < 1 or not 0 <= warmup < horizon:
|
| 74 |
+
raise ValueError("schedule horizon must exceed nonnegative warmup")
|
| 75 |
+
if step < warmup:
|
| 76 |
+
return (step + 1) / warmup
|
| 77 |
+
progress = min(1.0, max(0.0, (step - warmup) / (horizon - warmup)))
|
| 78 |
+
return 0.1 + 0.9 * (1 + math.cos(math.pi * progress)) / 2
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def per_sample_action_loss(prediction, target, mask):
|
| 82 |
+
"""Same group/sample weighting as v1_residual_objective, without reduction."""
|
| 83 |
+
safe_target = torch.where(mask, target, prediction.detach())
|
| 84 |
+
error = F.smooth_l1_loss(prediction, safe_target, reduction="none")
|
| 85 |
+
values, present = [], []
|
| 86 |
+
for lo, hi in ACTION_GROUPS:
|
| 87 |
+
count = mask[..., lo:hi].sum((1, 2))
|
| 88 |
+
values.append(
|
| 89 |
+
error[..., lo:hi].masked_fill(~mask[..., lo:hi], 0).sum((1, 2))
|
| 90 |
+
/ count.clamp_min(1)
|
| 91 |
+
)
|
| 92 |
+
present.append(count > 0)
|
| 93 |
+
occupied = torch.stack(present, 1)
|
| 94 |
+
if not occupied.any(1).all():
|
| 95 |
+
raise ValueError("every study sample must contain supervised actions")
|
| 96 |
+
return torch.stack(values, 1).sum(1) / occupied.sum(1)
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def masked_dynamics_loss(predicted, target, valid):
|
| 100 |
+
"""Exclude missing/ambiguous future labels before arithmetic, including NaNs."""
|
| 101 |
+
safe = torch.where(valid[..., None], target, predicted.detach())
|
| 102 |
+
values = F.smooth_l1_loss(predicted, safe, reduction="none").mean(-1)
|
| 103 |
+
return values.masked_fill(~valid, 0).sum() / valid.sum().clamp_min(1)
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
class IncrementalTouchEditor(nn.Module):
|
| 107 |
+
def __init__(self, action_mean, action_std, setting, hidden=128):
|
| 108 |
+
super().__init__()
|
| 109 |
+
self.setting = setting
|
| 110 |
+
self.register_buffer("action_mean", action_mean)
|
| 111 |
+
self.register_buffer("action_std", action_std.clamp_min(1e-3))
|
| 112 |
+
self.action_projection = nn.Linear(192, hidden)
|
| 113 |
+
# All settings reserve identical capacity for the optional innovation.
|
| 114 |
+
self.touch_projection = nn.Linear(128, hidden)
|
| 115 |
+
self.position = nn.Parameter(torch.randn(1, 16, hidden) * 0.02)
|
| 116 |
+
self.hand_embedding = nn.Parameter(torch.randn(1, 2, hidden) * 0.02)
|
| 117 |
+
self.time_projection = nn.Linear(1, hidden)
|
| 118 |
+
self.attention = GatedCrossAttention(hidden, 4)
|
| 119 |
+
nn.init.constant_(self.attention.gate, 0.1)
|
| 120 |
+
layer = nn.TransformerEncoderLayer(
|
| 121 |
+
hidden,
|
| 122 |
+
4,
|
| 123 |
+
hidden * 4,
|
| 124 |
+
dropout=setting.dropout,
|
| 125 |
+
batch_first=True,
|
| 126 |
+
norm_first=True,
|
| 127 |
+
)
|
| 128 |
+
self.trunk = nn.TransformerEncoder(layer, 1, enable_nested_tensor=False)
|
| 129 |
+
self.output = nn.Linear(hidden, 192)
|
| 130 |
+
nn.init.zeros_(self.output.weight)
|
| 131 |
+
nn.init.zeros_(self.output.bias)
|
| 132 |
+
|
| 133 |
+
def forward(
|
| 134 |
+
self, anchor, mask, tokens, hand_valid, elapsed, predicted_tokens, mean_tokens
|
| 135 |
+
):
|
| 136 |
+
condition = self.action_projection(
|
| 137 |
+
(anchor.detach() - self.action_mean) / self.action_std
|
| 138 |
+
)
|
| 139 |
+
if self.training and self.setting.condition_dropout:
|
| 140 |
+
# Block-mask plan steps, not residual anchors, targets or masks.
|
| 141 |
+
keep = (
|
| 142 |
+
torch.rand((*condition.shape[:2], 1), device=anchor.device)
|
| 143 |
+
>= self.setting.condition_dropout
|
| 144 |
+
)
|
| 145 |
+
condition = condition * keep
|
| 146 |
+
if self.setting.constant:
|
| 147 |
+
tokens = mean_tokens.expand_as(tokens)
|
| 148 |
+
if self.training and self.setting.touch_dropout:
|
| 149 |
+
keep = (
|
| 150 |
+
torch.rand((len(tokens), 1, 1), device=tokens.device)
|
| 151 |
+
>= self.setting.touch_dropout
|
| 152 |
+
)
|
| 153 |
+
tokens = torch.where(keep, tokens, mean_tokens.expand_as(tokens))
|
| 154 |
+
innovation = (
|
| 155 |
+
tokens - predicted_tokens
|
| 156 |
+
if self.setting.innovation
|
| 157 |
+
else torch.zeros_like(tokens)
|
| 158 |
+
)
|
| 159 |
+
memory = (
|
| 160 |
+
self.touch_projection(torch.cat((tokens, innovation), -1))
|
| 161 |
+
+ self.hand_embedding
|
| 162 |
+
)
|
| 163 |
+
query = (
|
| 164 |
+
condition
|
| 165 |
+
+ self.position
|
| 166 |
+
+ self.time_projection(elapsed.float()[:, None, None] * 10)
|
| 167 |
+
)
|
| 168 |
+
hidden = self.trunk(self.attention(query, memory, hand_valid))
|
| 169 |
+
raw_delta = self.output(hidden)
|
| 170 |
+
delta = raw_delta * mask * hand_valid.any(1)[:, None, None]
|
| 171 |
+
return anchor.detach() + delta, delta
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
def balance_condition_gradients(model):
|
| 175 |
+
"""Exploratory regression adaptation, NOT a reproduction of OGM-GE.
|
| 176 |
+
|
| 177 |
+
Only attenuate action-projection gradients when their norm exceeds the touch
|
| 178 |
+
projection norm. No VLA updates, classification scores, or added GE noise.
|
| 179 |
+
"""
|
| 180 |
+
|
| 181 |
+
def norm(module):
|
| 182 |
+
return sum(
|
| 183 |
+
(
|
| 184 |
+
p.grad.detach().square().sum()
|
| 185 |
+
for p in module.parameters()
|
| 186 |
+
if p.grad is not None
|
| 187 |
+
),
|
| 188 |
+
model.position.new_zeros(()),
|
| 189 |
+
).sqrt()
|
| 190 |
+
|
| 191 |
+
action, touch = norm(model.action_projection), norm(model.touch_projection)
|
| 192 |
+
factor = (touch / action.clamp_min(1e-12)).clamp(0.1, 1.0)
|
| 193 |
+
for p in model.action_projection.parameters():
|
| 194 |
+
if p.grad is not None:
|
| 195 |
+
p.grad.mul_(factor)
|
| 196 |
+
return float(factor), float(action), float(touch)
|
model/tachin_paired30_cache_20260923/code/src/tactile_editor/trainability.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Explicit trainable-component policies for controlled experiments."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from torch import nn
|
| 6 |
+
|
| 7 |
+
from .config import TrainMode, Variant
|
| 8 |
+
from .model import TactileMethodD, component_report
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def _set(module: nn.Module | None, enabled: bool) -> None:
|
| 12 |
+
if module is None:
|
| 13 |
+
return
|
| 14 |
+
for parameter in module.parameters():
|
| 15 |
+
parameter.requires_grad = enabled
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def configure_trainable_components(
|
| 19 |
+
model: TactileMethodD,
|
| 20 |
+
mode: TrainMode | str | None = None,
|
| 21 |
+
*,
|
| 22 |
+
expert_top_layers: int = 4,
|
| 23 |
+
) -> dict[str, dict[str, int]]:
|
| 24 |
+
"""Apply one parameter policy and return an auditable count by component."""
|
| 25 |
+
|
| 26 |
+
selected = TrainMode(mode or model.config.train_mode)
|
| 27 |
+
_set(model.method_d, False)
|
| 28 |
+
_set(model.tactile_encoder, False)
|
| 29 |
+
_set(model.fusion, False)
|
| 30 |
+
|
| 31 |
+
if selected == TrainMode.ADAPTER_ONLY:
|
| 32 |
+
_set(model.fusion, model.variant != Variant.V0_BASE)
|
| 33 |
+
elif selected == TrainMode.ENCODER_EDITOR:
|
| 34 |
+
if model.variant != Variant.V1_POSTHOC or model.tactile_encoder is None:
|
| 35 |
+
raise ValueError("encoder_editor requires V1 and a tactile encoder")
|
| 36 |
+
_set(model.tactile_encoder, True)
|
| 37 |
+
_set(model.fusion, True)
|
| 38 |
+
elif selected == TrainMode.EXPERT_TOP:
|
| 39 |
+
if expert_top_layers <= 0:
|
| 40 |
+
raise ValueError("expert_top_layers must be positive")
|
| 41 |
+
if expert_top_layers > len(model.method_d.expert.layers):
|
| 42 |
+
raise ValueError("expert_top_layers exceeds the expert depth")
|
| 43 |
+
_set(model.fusion, model.variant != Variant.V0_BASE)
|
| 44 |
+
layers = model.method_d.expert.layers
|
| 45 |
+
for layer in layers[-expert_top_layers:]:
|
| 46 |
+
_set(layer, True)
|
| 47 |
+
_set(model.method_d.expert.norm_out, True)
|
| 48 |
+
_set(model.method_d.expert.action_out_proj, True)
|
| 49 |
+
elif selected == TrainMode.ENCODER_TOP:
|
| 50 |
+
_set(model.fusion, model.variant != Variant.V0_BASE)
|
| 51 |
+
if model.tactile_encoder is None:
|
| 52 |
+
raise ValueError("encoder_top is not meaningful for V0")
|
| 53 |
+
# The published wrapper owns SelectedTactileEncoder as `.encoder`; that in
|
| 54 |
+
# turn owns the actual TactileEncoder as another `.encoder`.
|
| 55 |
+
selected_encoder = model.tactile_encoder.encoder
|
| 56 |
+
core = getattr(selected_encoder, "encoder", selected_encoder)
|
| 57 |
+
if not hasattr(core, "projection") or not hasattr(core, "gru"):
|
| 58 |
+
raise TypeError("encoder_top expects the published CNN+BiGRU encoder")
|
| 59 |
+
_set(core.projection, True)
|
| 60 |
+
for name, parameter in core.gru.named_parameters():
|
| 61 |
+
parameter.requires_grad = "_l1" in name
|
| 62 |
+
elif selected == TrainMode.JOINT:
|
| 63 |
+
_set(model.method_d, True)
|
| 64 |
+
_set(model.tactile_encoder, model.tactile_encoder is not None)
|
| 65 |
+
_set(model.fusion, model.fusion is not None)
|
| 66 |
+
else: # pragma: no cover - Enum makes this unreachable
|
| 67 |
+
raise ValueError(selected)
|
| 68 |
+
if not any(p.requires_grad for p in model.method_d.parameters()):
|
| 69 |
+
model.method_d.eval()
|
| 70 |
+
return component_report(model)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def encoder_editor_parameter_groups(
|
| 74 |
+
model: TactileMethodD, *, editor_lr: float, encoder_lr: float
|
| 75 |
+
) -> list[dict]:
|
| 76 |
+
"""Two disjoint optimizer groups; refuse accidental backbone updates."""
|
| 77 |
+
import math
|
| 78 |
+
|
| 79 |
+
if any(not math.isfinite(lr) or lr <= 0 for lr in (editor_lr, encoder_lr)):
|
| 80 |
+
raise ValueError("learning rates must be finite and positive")
|
| 81 |
+
if any(p.requires_grad for p in model.method_d.parameters()):
|
| 82 |
+
raise ValueError("Method D must remain frozen")
|
| 83 |
+
groups = []
|
| 84 |
+
for name, module, lr in (
|
| 85 |
+
("editor", model.fusion, editor_lr),
|
| 86 |
+
("encoder", model.tactile_encoder, encoder_lr),
|
| 87 |
+
):
|
| 88 |
+
params = (
|
| 89 |
+
[]
|
| 90 |
+
if module is None
|
| 91 |
+
else [p for p in module.parameters() if p.requires_grad]
|
| 92 |
+
)
|
| 93 |
+
if not params:
|
| 94 |
+
raise ValueError(f"{name} has no trainable parameters")
|
| 95 |
+
groups.append({"name": name, "params": params, "lr": lr})
|
| 96 |
+
ids = [id(p) for group in groups for p in group["params"]]
|
| 97 |
+
if len(ids) != len(set(ids)):
|
| 98 |
+
raise ValueError("optimizer groups overlap")
|
| 99 |
+
return groups
|
model/tachin_paired30_cache_20260923/code/src/tactile_editor/training.py
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""V1 training primitives derived from the Kai-final VLA timebase."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import math
|
| 6 |
+
from dataclasses import dataclass
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
import torch.nn.functional as F
|
| 10 |
+
|
| 11 |
+
from .action_layout import FULL_ACTION_GROUPS
|
| 12 |
+
from .model import TactileMethodD
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
# Backwards-compatible name used by existing objectives and scripts.
|
| 16 |
+
ACTION_GROUPS = FULL_ACTION_GROUPS
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def balanced_masked_mean(
|
| 20 |
+
values: torch.Tensor,
|
| 21 |
+
mask: torch.Tensor,
|
| 22 |
+
groups: tuple[tuple[int, int], ...] = ACTION_GROUPS,
|
| 23 |
+
) -> torch.Tensor:
|
| 24 |
+
"""Mean over values per group, present groups per sample, then valid samples.
|
| 25 |
+
|
| 26 |
+
Empty groups/samples are omitted; an entirely empty batch is an error.
|
| 27 |
+
Invalid values are selected away before reduction (including NaN padding).
|
| 28 |
+
"""
|
| 29 |
+
if values.ndim != 3 or mask.shape != values.shape:
|
| 30 |
+
raise ValueError("values and mask must have matching [B,T,A] shapes")
|
| 31 |
+
occupied: set[int] = set()
|
| 32 |
+
means, present = [], []
|
| 33 |
+
for start, end in groups:
|
| 34 |
+
if not 0 <= start < end <= values.shape[-1]:
|
| 35 |
+
raise ValueError("action group outside the action dimension")
|
| 36 |
+
indices = set(range(start, end))
|
| 37 |
+
if occupied & indices:
|
| 38 |
+
raise ValueError("action groups must not overlap")
|
| 39 |
+
occupied |= indices
|
| 40 |
+
selected = mask[..., start:end].bool()
|
| 41 |
+
count = selected.sum((1, 2))
|
| 42 |
+
means.append(
|
| 43 |
+
values[..., start:end].masked_fill(~selected, 0).sum((1, 2))
|
| 44 |
+
/ count.clamp_min(1)
|
| 45 |
+
)
|
| 46 |
+
present.append(count > 0)
|
| 47 |
+
if not groups:
|
| 48 |
+
raise ValueError("at least one action group is required")
|
| 49 |
+
valid = torch.stack(present, dim=1)
|
| 50 |
+
per_sample = torch.stack(means, dim=1).sum(1) / valid.sum(1).clamp_min(1)
|
| 51 |
+
nonempty = valid.any(1)
|
| 52 |
+
if not nonempty.any():
|
| 53 |
+
raise ValueError("the batch has no editable action values")
|
| 54 |
+
return per_sample[nonempty].mean()
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
@dataclass(frozen=True)
|
| 58 |
+
class V1TrainingBatch:
|
| 59 |
+
"""One cached Method-D plan and its synchronized tactile supervision."""
|
| 60 |
+
|
| 61 |
+
base_action: torch.Tensor # [B,16,192], normalized frozen-VLA output
|
| 62 |
+
target_action: torch.Tensor # [B,16,192], normalized ground truth
|
| 63 |
+
action_mask: torch.Tensor # [B,16,192]
|
| 64 |
+
touch_pressure: torch.Tensor # [B,8,2,21,21]
|
| 65 |
+
touch_valid: torch.Tensor # [B,8,2]
|
| 66 |
+
decision_after_start_seconds: torch.Tensor # [B], offline plan-relative time
|
| 67 |
+
|
| 68 |
+
def validate(self) -> None:
|
| 69 |
+
if self.base_action.ndim != 3:
|
| 70 |
+
raise ValueError("base_action must be [B,T,A]")
|
| 71 |
+
if self.target_action.shape != self.base_action.shape:
|
| 72 |
+
raise ValueError("target_action and base_action must have the same shape")
|
| 73 |
+
if self.action_mask.shape != self.base_action.shape:
|
| 74 |
+
raise ValueError("action_mask and base_action must have the same shape")
|
| 75 |
+
batch = len(self.base_action)
|
| 76 |
+
if self.touch_pressure.shape != (batch, 8, 2, 21, 21):
|
| 77 |
+
raise ValueError("touch_pressure must be [B,8,2,21,21]")
|
| 78 |
+
if self.touch_valid.shape != (batch, 8, 2):
|
| 79 |
+
raise ValueError("touch_valid must be [B,8,2]")
|
| 80 |
+
if self.decision_after_start_seconds.shape != (batch,):
|
| 81 |
+
raise ValueError("decision_after_start_seconds must be [B]")
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def build_future_action_mask(
|
| 85 |
+
action_mask: torch.Tensor,
|
| 86 |
+
decision_after_start_seconds: torch.Tensor,
|
| 87 |
+
*,
|
| 88 |
+
action_hz: float = 80.0,
|
| 89 |
+
) -> torch.Tensor:
|
| 90 |
+
"""Offline mask at the tactile decision time; no compute-latency model."""
|
| 91 |
+
|
| 92 |
+
if action_mask.ndim != 3:
|
| 93 |
+
raise ValueError("action_mask must be [B,T,A]")
|
| 94 |
+
if decision_after_start_seconds.shape != (len(action_mask),):
|
| 95 |
+
raise ValueError("decision_after_start_seconds must be [B]")
|
| 96 |
+
if not math.isfinite(action_hz) or action_hz <= 0:
|
| 97 |
+
raise ValueError("action_hz must be finite and positive")
|
| 98 |
+
ready = decision_after_start_seconds.to(
|
| 99 |
+
device=action_mask.device, dtype=torch.float64
|
| 100 |
+
)
|
| 101 |
+
if not torch.isfinite(ready).all() or (ready < 0).any():
|
| 102 |
+
raise ValueError("decision times must be finite and non-negative")
|
| 103 |
+
# Float32 seconds can put an exact tick a few ulps above its integer index.
|
| 104 |
+
tolerance = 1e-6 if decision_after_start_seconds.dtype != torch.float64 else 1e-9
|
| 105 |
+
committed = torch.ceil(ready * action_hz - tolerance).long()
|
| 106 |
+
committed = committed.clamp(max=action_mask.shape[1])
|
| 107 |
+
steps = torch.arange(action_mask.shape[1], device=action_mask.device)[None, :, None]
|
| 108 |
+
return action_mask.to(dtype=torch.bool) & (steps >= committed[:, None, None])
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def v1_residual_objective(
|
| 112 |
+
model: TactileMethodD,
|
| 113 |
+
batch: V1TrainingBatch,
|
| 114 |
+
*,
|
| 115 |
+
action_hz: float = 80.0,
|
| 116 |
+
delta_l2_weight: float = 1e-4,
|
| 117 |
+
action_groups: tuple[tuple[int, int], ...] = ACTION_GROUPS,
|
| 118 |
+
) -> tuple[torch.Tensor, dict[str, torch.Tensor]]:
|
| 119 |
+
"""Train V1 on only the action positions that deployment can still change."""
|
| 120 |
+
|
| 121 |
+
batch.validate()
|
| 122 |
+
if delta_l2_weight < 0:
|
| 123 |
+
raise ValueError("delta_l2_weight must be non-negative")
|
| 124 |
+
edit_mask = build_future_action_mask(
|
| 125 |
+
batch.action_mask,
|
| 126 |
+
batch.decision_after_start_seconds,
|
| 127 |
+
action_hz=action_hz,
|
| 128 |
+
)
|
| 129 |
+
# Exclude all dimensions outside the specified physical groups, even if the
|
| 130 |
+
# supplied mask accidentally marks padding as valid.
|
| 131 |
+
component_mask = torch.zeros_like(edit_mask)
|
| 132 |
+
for start, end in action_groups:
|
| 133 |
+
if not 0 <= start < end <= edit_mask.shape[-1]:
|
| 134 |
+
raise ValueError("action group outside the action dimension")
|
| 135 |
+
component_mask[..., start:end] = True
|
| 136 |
+
edit_mask &= component_mask
|
| 137 |
+
base = batch.base_action.detach()
|
| 138 |
+
target = batch.target_action.detach()
|
| 139 |
+
if not torch.isfinite(base).all():
|
| 140 |
+
raise ValueError("base_action must be finite, including padding")
|
| 141 |
+
if not torch.isfinite(target[edit_mask]).all():
|
| 142 |
+
raise ValueError("editable target_action must be finite")
|
| 143 |
+
# Invalid GT can contain NaN; remove it before Smooth L1 so backward is safe.
|
| 144 |
+
target = torch.where(edit_mask, target, base)
|
| 145 |
+
revised = model.edit_existing_plan(
|
| 146 |
+
base,
|
| 147 |
+
edit_mask,
|
| 148 |
+
batch.touch_pressure,
|
| 149 |
+
batch.touch_valid,
|
| 150 |
+
canonicalized_touch=False,
|
| 151 |
+
)
|
| 152 |
+
per_value = F.smooth_l1_loss(revised, target, reduction="none")
|
| 153 |
+
action_loss = balanced_masked_mean(per_value, edit_mask, action_groups)
|
| 154 |
+
delta = revised - base
|
| 155 |
+
delta_l2 = balanced_masked_mean(delta.square(), edit_mask, action_groups)
|
| 156 |
+
loss = action_loss + delta_l2_weight * delta_l2
|
| 157 |
+
with torch.no_grad():
|
| 158 |
+
base_error = balanced_masked_mean(
|
| 159 |
+
F.smooth_l1_loss(base, target, reduction="none"), edit_mask, action_groups
|
| 160 |
+
)
|
| 161 |
+
metrics = {
|
| 162 |
+
"loss": loss.detach(),
|
| 163 |
+
"action_loss": action_loss.detach(),
|
| 164 |
+
"delta_l2": delta_l2.detach(),
|
| 165 |
+
"base_action_loss": base_error.detach(),
|
| 166 |
+
"editable_fraction": edit_mask.float().mean().detach(),
|
| 167 |
+
"supervised_samples": edit_mask.any(dim=(1, 2)).sum().detach(),
|
| 168 |
+
}
|
| 169 |
+
return loss, metrics
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def nominal_revision_times(
|
| 173 |
+
*, tactile_hz: float = 30.0, control_hz: float = 10.0
|
| 174 |
+
) -> tuple[float, ...]:
|
| 175 |
+
"""Return tactile decision times inside one VLA control period."""
|
| 176 |
+
|
| 177 |
+
if tactile_hz <= 0 or control_hz <= 0:
|
| 178 |
+
raise ValueError("rates must be positive")
|
| 179 |
+
count = max(1, math.ceil(tactile_hz / control_hz))
|
| 180 |
+
return tuple(index / tactile_hz for index in range(count))
|
model/tachin_paired30_cache_20260923/code/src/tactile_editor/unified_cache.py
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Memory-mapped cache shared by the W0--W3 comparison."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import bisect
|
| 6 |
+
import hashlib
|
| 7 |
+
import json
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
import numpy as np
|
| 11 |
+
import torch
|
| 12 |
+
from torch.utils.data import Dataset
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
COMMON_ARRAYS = (
|
| 16 |
+
"base_action",
|
| 17 |
+
"target_action",
|
| 18 |
+
"action_mask",
|
| 19 |
+
"touch_pressure",
|
| 20 |
+
"touch_valid",
|
| 21 |
+
"decision_seconds",
|
| 22 |
+
"recording",
|
| 23 |
+
"final_memory",
|
| 24 |
+
"future_touch_pressure",
|
| 25 |
+
"future_touch_valid",
|
| 26 |
+
"future_source_frame",
|
| 27 |
+
)
|
| 28 |
+
|
| 29 |
+
RGB_ARRAYS = (
|
| 30 |
+
"image_tokens",
|
| 31 |
+
"clean_latent",
|
| 32 |
+
"rgb_valid",
|
| 33 |
+
"future_roi",
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
class UnifiedWorldCache(Dataset):
|
| 38 |
+
"""Read one immutable row set for every W0--W3 experiment.
|
| 39 |
+
|
| 40 |
+
A cache row is admitted only if it has an editable tactile observation, an
|
| 41 |
+
exact +0.1-s tactile target, and valid RGB targets/ROI. Even W0 and W2 use
|
| 42 |
+
this same intersection so their reported action scores are paired with W1
|
| 43 |
+
and W3 sample by sample.
|
| 44 |
+
"""
|
| 45 |
+
|
| 46 |
+
def __init__(
|
| 47 |
+
self,
|
| 48 |
+
root: str | Path,
|
| 49 |
+
split: str,
|
| 50 |
+
*,
|
| 51 |
+
include_rgb: bool,
|
| 52 |
+
expected_shards: int = 8,
|
| 53 |
+
) -> None:
|
| 54 |
+
self.root = Path(root)
|
| 55 |
+
self.split = split
|
| 56 |
+
self.include_rgb = include_rgb
|
| 57 |
+
self.names = COMMON_ARRAYS + (RGB_ARRAYS if include_rgb else ())
|
| 58 |
+
self.shards: list[tuple[int, dict[str, np.ndarray]]] = []
|
| 59 |
+
self.ends: list[int] = []
|
| 60 |
+
self.metadata: list[dict] = []
|
| 61 |
+
total = 0
|
| 62 |
+
for shard_root in sorted(self.root.glob("shard_*")):
|
| 63 |
+
split_root = shard_root / split
|
| 64 |
+
metadata_path = split_root / "metadata.json"
|
| 65 |
+
if not metadata_path.is_file():
|
| 66 |
+
continue
|
| 67 |
+
metadata = json.loads(metadata_path.read_text())
|
| 68 |
+
if int(metadata.get("cache_schema", -1)) != 3:
|
| 69 |
+
raise RuntimeError(f"not a unified W0--W3 cache: {split_root}")
|
| 70 |
+
if metadata.get("decision_ticks") != [0]:
|
| 71 |
+
raise RuntimeError("W0--W3 RGB comparison requires decision tick 0")
|
| 72 |
+
count = int(metadata["valid_samples"])
|
| 73 |
+
missing = [name for name in self.names if not (split_root / f"{name}.npy").is_file()]
|
| 74 |
+
if missing:
|
| 75 |
+
raise RuntimeError(f"missing {missing} in {split_root}")
|
| 76 |
+
arrays = {
|
| 77 |
+
name: np.load(split_root / f"{name}.npy", mmap_mode="r")
|
| 78 |
+
for name in self.names
|
| 79 |
+
}
|
| 80 |
+
if any(len(array) < count for array in arrays.values()):
|
| 81 |
+
raise RuntimeError(f"array shorter than metadata in {split_root}")
|
| 82 |
+
self.shards.append((count, arrays))
|
| 83 |
+
self.metadata.append(metadata)
|
| 84 |
+
total += count
|
| 85 |
+
self.ends.append(total)
|
| 86 |
+
if len(self.shards) != expected_shards:
|
| 87 |
+
raise RuntimeError(
|
| 88 |
+
f"expected {expected_shards} cache shards, found {len(self.shards)}"
|
| 89 |
+
)
|
| 90 |
+
fingerprints = {item["contract_fingerprint"] for item in self.metadata}
|
| 91 |
+
if len(fingerprints) != 1:
|
| 92 |
+
raise RuntimeError("cache shards have different data/model contracts")
|
| 93 |
+
self.contract_fingerprint = fingerprints.pop()
|
| 94 |
+
|
| 95 |
+
def __len__(self) -> int:
|
| 96 |
+
return self.ends[-1]
|
| 97 |
+
|
| 98 |
+
def __getitem__(self, index: int) -> dict[str, torch.Tensor]:
|
| 99 |
+
if index < 0:
|
| 100 |
+
index += len(self)
|
| 101 |
+
if not 0 <= index < len(self):
|
| 102 |
+
raise IndexError(index)
|
| 103 |
+
shard = bisect.bisect_right(self.ends, index)
|
| 104 |
+
start = 0 if shard == 0 else self.ends[shard - 1]
|
| 105 |
+
arrays = self.shards[shard][1]
|
| 106 |
+
result = {
|
| 107 |
+
name: torch.from_numpy(np.array(array[index - start], copy=True))
|
| 108 |
+
for name, array in arrays.items()
|
| 109 |
+
}
|
| 110 |
+
result["index"] = torch.tensor(index, dtype=torch.long)
|
| 111 |
+
return result
|
| 112 |
+
|
| 113 |
+
def identity(self) -> dict:
|
| 114 |
+
counts = [count for count, _ in self.shards]
|
| 115 |
+
payload = {
|
| 116 |
+
"root": str(self.root.resolve()),
|
| 117 |
+
"split": self.split,
|
| 118 |
+
"counts": counts,
|
| 119 |
+
"contract_fingerprint": self.contract_fingerprint,
|
| 120 |
+
}
|
| 121 |
+
canonical = json.dumps(payload, sort_keys=True).encode()
|
| 122 |
+
return {**payload, "identity_sha256": hashlib.sha256(canonical).hexdigest()}
|
model/tachin_paired30_cache_20260923/code/src/tactile_editor/visual_dynamics_pilot.py
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Cheap frozen-backbone pilot for visual world-model design choices.
|
| 2 |
+
|
| 3 |
+
This module is deliberately smaller than T-WLA. It consumes cached final-memory
|
| 4 |
+
tokens from the fixed ten-domain 110k checkpoint and trains one shared bottleneck
|
| 5 |
+
that serves both an action residual head and a visual-dynamics head. It is a
|
| 6 |
+
screening experiment: a positive result justifies an expensive end-to-end run; a
|
| 7 |
+
negative result prevents spending that budget.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
from dataclasses import dataclass
|
| 13 |
+
|
| 14 |
+
import torch
|
| 15 |
+
from torch import nn
|
| 16 |
+
|
| 17 |
+
from .action_layout import DEFAULT_ACTION_LAYOUT, FULL_ACTION_GROUPS, VALID_ACTION_DIM
|
| 18 |
+
from .action_objectives import per_sample_action_loss
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@dataclass
|
| 22 |
+
class VisualDynamicsOutput:
|
| 23 |
+
action: torch.Tensor
|
| 24 |
+
delta: torch.Tensor
|
| 25 |
+
shared: torch.Tensor
|
| 26 |
+
future_latent: torch.Tensor | None
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class VisualDynamicsPilot(nn.Module):
|
| 30 |
+
"""Shared memory adapter with optional action-conditioned visual dynamics.
|
| 31 |
+
|
| 32 |
+
``world_condition`` is one of:
|
| 33 |
+
|
| 34 |
+
* ``none``: action-only control; no visual head is evaluated.
|
| 35 |
+
* ``memory``: the visual head reads the shared memory bottleneck only.
|
| 36 |
+
* ``predicted_action``: it additionally reads the differentiable action
|
| 37 |
+
prediction, so the visual loss can directly regulate the action head.
|
| 38 |
+
"""
|
| 39 |
+
|
| 40 |
+
def __init__(
|
| 41 |
+
self,
|
| 42 |
+
*,
|
| 43 |
+
memory_tokens: int = 8,
|
| 44 |
+
memory_dim: int = 960,
|
| 45 |
+
hidden_dim: int = 384,
|
| 46 |
+
action_hidden_dim: int = 128,
|
| 47 |
+
time_dim: int = 32,
|
| 48 |
+
horizons: int = 2,
|
| 49 |
+
latent_channels: int = 32,
|
| 50 |
+
latent_size: int = 16,
|
| 51 |
+
residual_limit: float = 0.5,
|
| 52 |
+
world_condition: str = "none",
|
| 53 |
+
world_step_ranges: tuple[tuple[int, int], ...] | None = None,
|
| 54 |
+
) -> None:
|
| 55 |
+
super().__init__()
|
| 56 |
+
if world_condition not in {
|
| 57 |
+
"none", "memory", "predicted_action", "predicted_action_only",
|
| 58 |
+
}:
|
| 59 |
+
raise ValueError(f"unknown world_condition: {world_condition}")
|
| 60 |
+
if residual_limit <= 0:
|
| 61 |
+
raise ValueError("residual_limit must be positive")
|
| 62 |
+
self.world_condition = world_condition
|
| 63 |
+
self.residual_limit = float(residual_limit)
|
| 64 |
+
self.horizons = horizons
|
| 65 |
+
self.latent_channels = latent_channels
|
| 66 |
+
self.latent_size = latent_size
|
| 67 |
+
self.world_step_ranges = world_step_ranges or tuple(
|
| 68 |
+
(0, steps) for steps in (8, 16)[:horizons]
|
| 69 |
+
)
|
| 70 |
+
if len(self.world_step_ranges) != horizons:
|
| 71 |
+
raise ValueError("one world action-step range is required per horizon")
|
| 72 |
+
if any(start < 0 or stop <= start or stop > 16 for start, stop in self.world_step_ranges):
|
| 73 |
+
raise ValueError("world action-step ranges must satisfy 0 <= start < stop <= 16")
|
| 74 |
+
|
| 75 |
+
self.memory_norm = nn.LayerNorm(memory_dim)
|
| 76 |
+
self.memory_encoder = nn.Sequential(
|
| 77 |
+
nn.Linear(memory_tokens * memory_dim, hidden_dim),
|
| 78 |
+
nn.GELU(),
|
| 79 |
+
nn.LayerNorm(hidden_dim),
|
| 80 |
+
nn.Linear(hidden_dim, hidden_dim),
|
| 81 |
+
nn.GELU(),
|
| 82 |
+
)
|
| 83 |
+
self.action_time = nn.Parameter(torch.zeros(16, time_dim))
|
| 84 |
+
self.action_head = nn.Sequential(
|
| 85 |
+
nn.Linear(hidden_dim + time_dim + VALID_ACTION_DIM, hidden_dim),
|
| 86 |
+
nn.GELU(),
|
| 87 |
+
nn.Linear(hidden_dim, VALID_ACTION_DIM),
|
| 88 |
+
)
|
| 89 |
+
|
| 90 |
+
if world_condition != "none":
|
| 91 |
+
self.action_encoder = nn.Sequential(
|
| 92 |
+
nn.Linear(VALID_ACTION_DIM, action_hidden_dim),
|
| 93 |
+
nn.GELU(),
|
| 94 |
+
nn.Linear(action_hidden_dim, action_hidden_dim),
|
| 95 |
+
)
|
| 96 |
+
self.horizon_embedding = nn.Parameter(torch.zeros(horizons, time_dim))
|
| 97 |
+
self.world_decoder = nn.Sequential(
|
| 98 |
+
nn.Linear(hidden_dim + action_hidden_dim + time_dim, 512),
|
| 99 |
+
nn.GELU(),
|
| 100 |
+
nn.Linear(512, latent_channels * latent_size * latent_size),
|
| 101 |
+
)
|
| 102 |
+
|
| 103 |
+
def forward(
|
| 104 |
+
self,
|
| 105 |
+
base_action: torch.Tensor,
|
| 106 |
+
action_mask: torch.Tensor,
|
| 107 |
+
final_memory: torch.Tensor,
|
| 108 |
+
*,
|
| 109 |
+
compute_world: bool = True,
|
| 110 |
+
) -> VisualDynamicsOutput:
|
| 111 |
+
DEFAULT_ACTION_LAYOUT.validate(base_action, name="base_action")
|
| 112 |
+
DEFAULT_ACTION_LAYOUT.validate(action_mask, name="action_mask")
|
| 113 |
+
if final_memory.shape != (len(base_action), 8, 960):
|
| 114 |
+
raise ValueError("final_memory must be [B,8,960]")
|
| 115 |
+
|
| 116 |
+
memory = self.memory_norm(final_memory.float()).flatten(1)
|
| 117 |
+
shared = self.memory_encoder(memory)
|
| 118 |
+
times = self.action_time[None].expand(len(base_action), -1, -1)
|
| 119 |
+
shared_steps = shared[:, None].expand(-1, 16, -1)
|
| 120 |
+
action_input = torch.cat(
|
| 121 |
+
[base_action[..., :VALID_ACTION_DIM].float(), shared_steps, times], dim=-1
|
| 122 |
+
)
|
| 123 |
+
raw_delta = self.residual_limit * torch.tanh(self.action_head(action_input))
|
| 124 |
+
physical_mask = action_mask.bool().clone()
|
| 125 |
+
physical_mask[..., VALID_ACTION_DIM:] = False
|
| 126 |
+
delta = torch.zeros_like(base_action.float())
|
| 127 |
+
delta[..., :VALID_ACTION_DIM] = raw_delta
|
| 128 |
+
delta = delta * physical_mask.to(delta.dtype)
|
| 129 |
+
action = base_action.float().detach() + delta
|
| 130 |
+
|
| 131 |
+
future = None
|
| 132 |
+
if self.world_condition != "none" and compute_world:
|
| 133 |
+
if self.world_condition in {"predicted_action", "predicted_action_only"}:
|
| 134 |
+
step_features = self.action_encoder(action[..., :VALID_ACTION_DIM])
|
| 135 |
+
valid_steps = physical_mask[..., :VALID_ACTION_DIM].any(-1)
|
| 136 |
+
else:
|
| 137 |
+
step_features = torch.zeros(
|
| 138 |
+
len(action), 16, self.action_encoder[-1].out_features,
|
| 139 |
+
device=action.device, dtype=shared.dtype,
|
| 140 |
+
)
|
| 141 |
+
valid_steps = torch.ones(
|
| 142 |
+
len(action), 16, device=action.device, dtype=torch.bool
|
| 143 |
+
)
|
| 144 |
+
|
| 145 |
+
predictions = []
|
| 146 |
+
for horizon, (start, stop) in enumerate(self.world_step_ranges):
|
| 147 |
+
valid = valid_steps[:, start:stop].to(step_features.dtype)
|
| 148 |
+
context = (
|
| 149 |
+
step_features[:, start:stop] * valid[..., None]
|
| 150 |
+
).sum(1) / valid.sum(1, keepdim=True).clamp_min(1)
|
| 151 |
+
world_memory = (
|
| 152 |
+
torch.zeros_like(shared)
|
| 153 |
+
if self.world_condition == "predicted_action_only"
|
| 154 |
+
else shared
|
| 155 |
+
)
|
| 156 |
+
condition = torch.cat(
|
| 157 |
+
[
|
| 158 |
+
world_memory,
|
| 159 |
+
context,
|
| 160 |
+
self.horizon_embedding[horizon][None].expand(len(action), -1),
|
| 161 |
+
],
|
| 162 |
+
dim=-1,
|
| 163 |
+
)
|
| 164 |
+
predictions.append(
|
| 165 |
+
self.world_decoder(condition).reshape(
|
| 166 |
+
len(action), self.latent_channels, self.latent_size, self.latent_size
|
| 167 |
+
)
|
| 168 |
+
)
|
| 169 |
+
future = torch.stack(predictions, dim=1)
|
| 170 |
+
return VisualDynamicsOutput(action, delta, shared, future)
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def visual_dynamics_losses(
|
| 174 |
+
output: VisualDynamicsOutput,
|
| 175 |
+
target_action: torch.Tensor,
|
| 176 |
+
action_mask: torch.Tensor,
|
| 177 |
+
*,
|
| 178 |
+
future_latent: torch.Tensor | None = None,
|
| 179 |
+
future_valid: torch.Tensor | None = None,
|
| 180 |
+
future_roi: torch.Tensor | None = None,
|
| 181 |
+
background_weight: float = 0.05,
|
| 182 |
+
delta_l2_weight: float = 1e-4,
|
| 183 |
+
) -> dict[str, torch.Tensor]:
|
| 184 |
+
"""Return action and ROI-weighted visual losses without combining them."""
|
| 185 |
+
|
| 186 |
+
mask = action_mask.bool().clone()
|
| 187 |
+
mask[..., VALID_ACTION_DIM:] = False
|
| 188 |
+
action = per_sample_action_loss(
|
| 189 |
+
output.action, target_action.float(), mask, groups=FULL_ACTION_GROUPS
|
| 190 |
+
).mean()
|
| 191 |
+
delta_l2 = output.delta.square().masked_fill(~mask, 0).sum() / mask.sum().clamp_min(1)
|
| 192 |
+
action_total = action + delta_l2_weight * delta_l2
|
| 193 |
+
|
| 194 |
+
world = action.new_zeros(())
|
| 195 |
+
if output.future_latent is not None:
|
| 196 |
+
if future_latent is None or future_valid is None or future_roi is None:
|
| 197 |
+
raise ValueError("future latent, validity, and ROI are required")
|
| 198 |
+
if future_latent.shape != output.future_latent.shape:
|
| 199 |
+
raise ValueError("future latent shape does not match the prediction")
|
| 200 |
+
if not 0 <= background_weight <= 1:
|
| 201 |
+
raise ValueError("background_weight must be in [0,1]")
|
| 202 |
+
valid = future_valid.bool()
|
| 203 |
+
roi = future_roi.bool()
|
| 204 |
+
weights = background_weight + (1.0 - background_weight) * roi.float()
|
| 205 |
+
weights = weights * valid[:, :, None, None].float()
|
| 206 |
+
squared = (output.future_latent.float() - future_latent.float()).square()
|
| 207 |
+
numerator = (squared * weights[:, :, None]).sum()
|
| 208 |
+
denominator = weights.sum() * squared.shape[2]
|
| 209 |
+
world = numerator / denominator.clamp_min(1)
|
| 210 |
+
return {
|
| 211 |
+
"action": action,
|
| 212 |
+
"action_total": action_total,
|
| 213 |
+
"delta_l2": delta_l2,
|
| 214 |
+
"world": world,
|
| 215 |
+
}
|
model/tachin_paired30_cache_20260923/code/src/tactile_editor/world_residual_probe.py
ADDED
|
@@ -0,0 +1,362 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Leakage-safe statistics for the Task-35 world/action residual pilot.
|
| 2 |
+
|
| 3 |
+
The module deliberately contains no model-training code. It consumes residuals
|
| 4 |
+
materialized by a frozen Stage-1 action calibrator and a frozen tactile-free W0
|
| 5 |
+
world model, then fits every transform on train only.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
from dataclasses import dataclass
|
| 11 |
+
import math
|
| 12 |
+
|
| 13 |
+
import numpy as np
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
LEFT_ARTICULATION = slice(6, 69)
|
| 17 |
+
RIGHT_ARTICULATION = slice(75, 138)
|
| 18 |
+
LEFT_HAND = slice(0, 69)
|
| 19 |
+
RIGHT_HAND = slice(69, 138)
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def rectified_flow_sample(
|
| 23 |
+
clean: np.ndarray, noise: np.ndarray, tau: np.ndarray
|
| 24 |
+
) -> tuple[np.ndarray, np.ndarray]:
|
| 25 |
+
"""Return z_tau and the clean-to-noise velocity under the V4 convention."""
|
| 26 |
+
|
| 27 |
+
clean = np.asarray(clean)
|
| 28 |
+
noise = np.asarray(noise)
|
| 29 |
+
tau = np.asarray(tau)
|
| 30 |
+
if clean.shape != noise.shape:
|
| 31 |
+
raise ValueError("clean and noise must have identical shapes")
|
| 32 |
+
if np.any(~np.isfinite(tau)) or np.any((tau < 0) | (tau > 1)):
|
| 33 |
+
raise ValueError("tau must be finite and in [0,1]")
|
| 34 |
+
while tau.ndim < clean.ndim:
|
| 35 |
+
tau = tau[..., None]
|
| 36 |
+
return (1.0 - tau) * clean + tau * noise, noise - clean
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def action_window_start(decision_seconds: np.ndarray, action_hz: float = 80.0) -> np.ndarray:
|
| 40 |
+
"""Match ``build_future_action_mask`` including its float32 tolerance."""
|
| 41 |
+
|
| 42 |
+
seconds = np.asarray(decision_seconds)
|
| 43 |
+
if np.any(~np.isfinite(seconds)) or np.any(seconds < 0):
|
| 44 |
+
raise ValueError("decision_seconds must be finite and non-negative")
|
| 45 |
+
tolerance = 1e-6 if seconds.dtype != np.float64 else 1e-9
|
| 46 |
+
return np.ceil(seconds.astype(np.float64) * action_hz - tolerance).astype(np.int64)
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def pool_visual_residual(residual: np.ndarray, output_size: int = 4) -> np.ndarray:
|
| 50 |
+
"""Average-pool ``[...,C,H,W]`` residual maps and flatten non-batch axes."""
|
| 51 |
+
|
| 52 |
+
value = np.asarray(residual, dtype=np.float32)
|
| 53 |
+
if value.ndim < 4:
|
| 54 |
+
raise ValueError("visual residual must end in [C,H,W]")
|
| 55 |
+
height, width = value.shape[-2:]
|
| 56 |
+
if height % output_size or width % output_size:
|
| 57 |
+
raise ValueError("spatial dimensions must be divisible by output_size")
|
| 58 |
+
value = value.reshape(
|
| 59 |
+
*value.shape[:-2], output_size, height // output_size,
|
| 60 |
+
output_size, width // output_size,
|
| 61 |
+
).mean(axis=(-3, -1))
|
| 62 |
+
return value.reshape(len(value), -1)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
@dataclass(frozen=True)
|
| 66 |
+
class HandRows:
|
| 67 |
+
visual: np.ndarray
|
| 68 |
+
articulation: np.ndarray
|
| 69 |
+
hand_action: np.ndarray
|
| 70 |
+
source_row: np.ndarray
|
| 71 |
+
hand: np.ndarray
|
| 72 |
+
recording: np.ndarray
|
| 73 |
+
timestamp: np.ndarray
|
| 74 |
+
subsets: dict[str, np.ndarray]
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def build_hand_rows(
|
| 78 |
+
visual: np.ndarray,
|
| 79 |
+
anchor_action: np.ndarray,
|
| 80 |
+
target_action: np.ndarray,
|
| 81 |
+
action_mask: np.ndarray,
|
| 82 |
+
decision_seconds: np.ndarray,
|
| 83 |
+
recording: np.ndarray,
|
| 84 |
+
*,
|
| 85 |
+
absolute_seconds: np.ndarray | None = None,
|
| 86 |
+
event_masks: dict[str, np.ndarray] | None = None,
|
| 87 |
+
horizon: int = 8,
|
| 88 |
+
action_hz: float = 80.0,
|
| 89 |
+
) -> HandRows:
|
| 90 |
+
"""Create one fully observed, time-aligned row per supervised hand.
|
| 91 |
+
|
| 92 |
+
A decision at 33.3 ms starts at action index 3, not index 0. The selected
|
| 93 |
+
eight-step residual therefore always covers the same physical 0.1-second
|
| 94 |
+
interval as the future RGB target.
|
| 95 |
+
"""
|
| 96 |
+
|
| 97 |
+
visual = np.asarray(visual, dtype=np.float32)
|
| 98 |
+
anchor = np.asarray(anchor_action, dtype=np.float32)
|
| 99 |
+
target = np.asarray(target_action, dtype=np.float32)
|
| 100 |
+
mask = np.asarray(action_mask, dtype=bool)
|
| 101 |
+
seconds = np.asarray(decision_seconds)
|
| 102 |
+
timestamps = seconds if absolute_seconds is None else np.asarray(absolute_seconds)
|
| 103 |
+
recordings = np.asarray(recording)
|
| 104 |
+
n = len(visual)
|
| 105 |
+
if anchor.shape != target.shape or anchor.shape != mask.shape:
|
| 106 |
+
raise ValueError("anchor, target, and mask must have identical shapes")
|
| 107 |
+
if anchor.ndim != 3 or anchor.shape[1:] != (16, 192):
|
| 108 |
+
raise ValueError("actions must be [N,16,192]")
|
| 109 |
+
if any(len(value) != n for value in (anchor, seconds, timestamps, recordings)):
|
| 110 |
+
raise ValueError("all arrays must have the same leading dimension")
|
| 111 |
+
starts = action_window_start(seconds, action_hz)
|
| 112 |
+
specs = ((LEFT_ARTICULATION, LEFT_HAND), (RIGHT_ARTICULATION, RIGHT_HAND))
|
| 113 |
+
rows: list[int] = []
|
| 114 |
+
hands: list[int] = []
|
| 115 |
+
articulation: list[np.ndarray] = []
|
| 116 |
+
hand_action: list[np.ndarray] = []
|
| 117 |
+
for row, start in enumerate(starts):
|
| 118 |
+
stop = int(start) + horizon
|
| 119 |
+
if stop > anchor.shape[1]:
|
| 120 |
+
continue
|
| 121 |
+
for hand, (art_slice, hand_slice) in enumerate(specs):
|
| 122 |
+
# CCA and ordinary ridge require a fixed fully observed target.
|
| 123 |
+
if not mask[row, start:stop, art_slice].all():
|
| 124 |
+
continue
|
| 125 |
+
residual = target[row, start:stop] - anchor[row, start:stop]
|
| 126 |
+
rows.append(row)
|
| 127 |
+
hands.append(hand)
|
| 128 |
+
articulation.append(residual[:, art_slice].reshape(-1))
|
| 129 |
+
hand_action.append(residual[:, hand_slice].reshape(-1))
|
| 130 |
+
if not rows:
|
| 131 |
+
raise ValueError("no hand has a fully valid aligned action horizon")
|
| 132 |
+
source = np.asarray(rows, dtype=np.int64)
|
| 133 |
+
hand_index = np.asarray(hands, dtype=np.int8)
|
| 134 |
+
subsets: dict[str, np.ndarray] = {}
|
| 135 |
+
for name, values in (event_masks or {}).items():
|
| 136 |
+
value = np.asarray(values, dtype=bool)
|
| 137 |
+
if value.shape == (n,):
|
| 138 |
+
subsets[name] = value[source]
|
| 139 |
+
elif value.shape == (n, 2):
|
| 140 |
+
subsets[name] = value[source, hand_index]
|
| 141 |
+
else:
|
| 142 |
+
raise ValueError(f"event mask {name!r} must be [N] or [N,2]")
|
| 143 |
+
return HandRows(
|
| 144 |
+
visual=visual[source],
|
| 145 |
+
articulation=np.stack(articulation),
|
| 146 |
+
hand_action=np.stack(hand_action),
|
| 147 |
+
source_row=source,
|
| 148 |
+
hand=hand_index,
|
| 149 |
+
recording=recordings[source],
|
| 150 |
+
timestamp=timestamps[source],
|
| 151 |
+
subsets=subsets,
|
| 152 |
+
)
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
@dataclass(frozen=True)
|
| 156 |
+
class PCATransform:
|
| 157 |
+
mean: np.ndarray
|
| 158 |
+
scale: np.ndarray
|
| 159 |
+
components: np.ndarray
|
| 160 |
+
explained_variance: np.ndarray
|
| 161 |
+
|
| 162 |
+
def transform(self, value: np.ndarray) -> np.ndarray:
|
| 163 |
+
standardized = (np.asarray(value, dtype=np.float64) - self.mean) / self.scale
|
| 164 |
+
return standardized @ self.components
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
def fit_pca(
|
| 168 |
+
value: np.ndarray,
|
| 169 |
+
components: int = 64,
|
| 170 |
+
*,
|
| 171 |
+
seed: int = 20260911,
|
| 172 |
+
oversample: int = 12,
|
| 173 |
+
power_iterations: int = 2,
|
| 174 |
+
) -> PCATransform:
|
| 175 |
+
"""Fit randomized PCA without materializing a feature covariance matrix."""
|
| 176 |
+
|
| 177 |
+
x = np.asarray(value, dtype=np.float64)
|
| 178 |
+
if x.ndim != 2 or len(x) < 3 or not np.isfinite(x).all():
|
| 179 |
+
raise ValueError("PCA input must be a finite [N,D] matrix")
|
| 180 |
+
mean = x.mean(0)
|
| 181 |
+
scale = x.std(0, ddof=1)
|
| 182 |
+
scale[scale < 1e-8] = 1.0
|
| 183 |
+
z = (x - mean) / scale
|
| 184 |
+
rank = min(int(components), len(x) - 1, x.shape[1])
|
| 185 |
+
sketch = min(rank + oversample, len(x), x.shape[1])
|
| 186 |
+
rng = np.random.default_rng(seed)
|
| 187 |
+
omega = rng.standard_normal((x.shape[1], sketch))
|
| 188 |
+
q = z @ omega
|
| 189 |
+
for _ in range(power_iterations):
|
| 190 |
+
q = z @ (z.T @ q)
|
| 191 |
+
q, _ = np.linalg.qr(q, mode="reduced")
|
| 192 |
+
_, singular, vt = np.linalg.svd(q.T @ z, full_matrices=False)
|
| 193 |
+
basis = vt[:rank].T
|
| 194 |
+
variance = singular[:rank] ** 2 / max(len(x) - 1, 1)
|
| 195 |
+
return PCATransform(mean, scale, basis, variance)
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
def _inverse_sqrt(matrix: np.ndarray) -> np.ndarray:
|
| 199 |
+
values, vectors = np.linalg.eigh(matrix)
|
| 200 |
+
values = np.maximum(values, 1e-12)
|
| 201 |
+
return (vectors / np.sqrt(values)[None, :]) @ vectors.T
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
@dataclass(frozen=True)
|
| 205 |
+
class CCAModel:
|
| 206 |
+
x_mean: np.ndarray
|
| 207 |
+
y_mean: np.ndarray
|
| 208 |
+
x_weights: np.ndarray
|
| 209 |
+
y_weights: np.ndarray
|
| 210 |
+
singular_values: np.ndarray
|
| 211 |
+
ridge: float
|
| 212 |
+
|
| 213 |
+
def transform(self, x: np.ndarray, y: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
|
| 214 |
+
return (
|
| 215 |
+
(np.asarray(x, dtype=np.float64) - self.x_mean) @ self.x_weights,
|
| 216 |
+
(np.asarray(y, dtype=np.float64) - self.y_mean) @ self.y_weights,
|
| 217 |
+
)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
def fit_regularized_cca(
|
| 221 |
+
x: np.ndarray, y: np.ndarray, *, ridge: float, components: int = 16
|
| 222 |
+
) -> CCAModel:
|
| 223 |
+
x = np.asarray(x, dtype=np.float64)
|
| 224 |
+
y = np.asarray(y, dtype=np.float64)
|
| 225 |
+
if x.ndim != 2 or y.ndim != 2 or len(x) != len(y) or len(x) < 3:
|
| 226 |
+
raise ValueError("CCA inputs must be paired [N,D] matrices")
|
| 227 |
+
xm, ym = x.mean(0), y.mean(0)
|
| 228 |
+
xc, yc = x - xm, y - ym
|
| 229 |
+
denominator = max(len(x) - 1, 1)
|
| 230 |
+
cxx = xc.T @ xc / denominator + ridge * np.eye(x.shape[1])
|
| 231 |
+
cyy = yc.T @ yc / denominator + ridge * np.eye(y.shape[1])
|
| 232 |
+
cxy = xc.T @ yc / denominator
|
| 233 |
+
inv_x = _inverse_sqrt(cxx)
|
| 234 |
+
inv_y = _inverse_sqrt(cyy)
|
| 235 |
+
left, singular, right_t = np.linalg.svd(inv_x @ cxy @ inv_y, full_matrices=False)
|
| 236 |
+
rank = min(components, len(singular))
|
| 237 |
+
return CCAModel(
|
| 238 |
+
xm, ym, inv_x @ left[:, :rank], inv_y @ right_t.T[:, :rank],
|
| 239 |
+
singular[:rank], float(ridge),
|
| 240 |
+
)
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
def canonical_correlations(x_scores: np.ndarray, y_scores: np.ndarray) -> np.ndarray:
|
| 244 |
+
if x_scores.shape != y_scores.shape:
|
| 245 |
+
raise ValueError("canonical score matrices must match")
|
| 246 |
+
x = x_scores - x_scores.mean(0)
|
| 247 |
+
y = y_scores - y_scores.mean(0)
|
| 248 |
+
numerator = (x * y).sum(0)
|
| 249 |
+
denominator = np.sqrt(((x * x).sum(0)) * ((y * y).sum(0)))
|
| 250 |
+
return np.divide(numerator, denominator, out=np.zeros_like(numerator), where=denominator > 0)
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
@dataclass(frozen=True)
|
| 254 |
+
class RidgeModel:
|
| 255 |
+
x_mean: np.ndarray
|
| 256 |
+
y_mean: np.ndarray
|
| 257 |
+
weights: np.ndarray
|
| 258 |
+
alpha: float
|
| 259 |
+
|
| 260 |
+
def predict(self, x: np.ndarray) -> np.ndarray:
|
| 261 |
+
return (np.asarray(x, dtype=np.float64) - self.x_mean) @ self.weights + self.y_mean
|
| 262 |
+
|
| 263 |
+
|
| 264 |
+
def fit_ridge(x: np.ndarray, y: np.ndarray, alpha: float) -> RidgeModel:
|
| 265 |
+
x = np.asarray(x, dtype=np.float64)
|
| 266 |
+
y = np.asarray(y, dtype=np.float64)
|
| 267 |
+
xm, ym = x.mean(0), y.mean(0)
|
| 268 |
+
xc, yc = x - xm, y - ym
|
| 269 |
+
gram = xc.T @ xc + alpha * np.eye(x.shape[1])
|
| 270 |
+
weights = np.linalg.solve(gram, xc.T @ yc)
|
| 271 |
+
return RidgeModel(xm, ym, weights, float(alpha))
|
| 272 |
+
|
| 273 |
+
|
| 274 |
+
def r2_scores(target: np.ndarray, prediction: np.ndarray) -> dict[str, float | list[float]]:
|
| 275 |
+
target = np.asarray(target, dtype=np.float64)
|
| 276 |
+
prediction = np.asarray(prediction, dtype=np.float64)
|
| 277 |
+
if target.shape != prediction.shape or target.ndim != 2:
|
| 278 |
+
raise ValueError("target and prediction must be matching [N,D] matrices")
|
| 279 |
+
residual = ((target - prediction) ** 2).sum(0)
|
| 280 |
+
total = ((target - target.mean(0)) ** 2).sum(0)
|
| 281 |
+
per_dimension = np.divide(
|
| 282 |
+
residual, total, out=np.full_like(residual, np.nan), where=total > 1e-12
|
| 283 |
+
)
|
| 284 |
+
per_dimension = 1.0 - per_dimension
|
| 285 |
+
valid = np.isfinite(per_dimension)
|
| 286 |
+
uniform = float(per_dimension[valid].mean()) if valid.any() else math.nan
|
| 287 |
+
variance_weighted = float(1.0 - residual.sum() / total.sum()) if total.sum() else math.nan
|
| 288 |
+
return {
|
| 289 |
+
"uniform_average": uniform,
|
| 290 |
+
"variance_weighted": variance_weighted,
|
| 291 |
+
"per_dimension": per_dimension.tolist(),
|
| 292 |
+
}
|
| 293 |
+
|
| 294 |
+
|
| 295 |
+
def within_recording_donors(
|
| 296 |
+
recording: np.ndarray,
|
| 297 |
+
hand: np.ndarray,
|
| 298 |
+
timestamp: np.ndarray,
|
| 299 |
+
*,
|
| 300 |
+
seed: int,
|
| 301 |
+
minimum_separation: float = 1.0,
|
| 302 |
+
) -> tuple[np.ndarray, np.ndarray]:
|
| 303 |
+
"""Sample same-recording/same-hand donors at least one second away."""
|
| 304 |
+
|
| 305 |
+
recording = np.asarray(recording)
|
| 306 |
+
hand = np.asarray(hand)
|
| 307 |
+
timestamp = np.asarray(timestamp, dtype=np.float64)
|
| 308 |
+
rng = np.random.default_rng(seed)
|
| 309 |
+
donors = np.arange(len(recording), dtype=np.int64)
|
| 310 |
+
eligible = np.zeros(len(recording), dtype=bool)
|
| 311 |
+
groups: dict[tuple[object, int], list[int]] = {}
|
| 312 |
+
for row, key in enumerate(zip(recording.tolist(), hand.tolist())):
|
| 313 |
+
groups.setdefault(key, []).append(row)
|
| 314 |
+
for indices in groups.values():
|
| 315 |
+
group = np.asarray(indices, dtype=np.int64)
|
| 316 |
+
times = timestamp[group]
|
| 317 |
+
for local, row in enumerate(group):
|
| 318 |
+
candidates = group[np.abs(times - times[local]) >= minimum_separation]
|
| 319 |
+
if len(candidates):
|
| 320 |
+
donors[row] = int(rng.choice(candidates))
|
| 321 |
+
eligible[row] = True
|
| 322 |
+
return donors, eligible
|
| 323 |
+
|
| 324 |
+
|
| 325 |
+
def cross_recording_donors(
|
| 326 |
+
recording: np.ndarray, hand: np.ndarray, *, seed: int
|
| 327 |
+
) -> np.ndarray:
|
| 328 |
+
"""Sample a different-recording donor with the same hand identity."""
|
| 329 |
+
|
| 330 |
+
recording = np.asarray(recording)
|
| 331 |
+
hand = np.asarray(hand)
|
| 332 |
+
rng = np.random.default_rng(seed)
|
| 333 |
+
donors = np.empty(len(recording), dtype=np.int64)
|
| 334 |
+
by_hand = {value: np.flatnonzero(hand == value) for value in np.unique(hand)}
|
| 335 |
+
for row in range(len(recording)):
|
| 336 |
+
candidates = by_hand[hand[row]]
|
| 337 |
+
candidates = candidates[recording[candidates] != recording[row]]
|
| 338 |
+
if not len(candidates):
|
| 339 |
+
raise ValueError("no cross-recording donor with matching hand")
|
| 340 |
+
donors[row] = int(rng.choice(candidates))
|
| 341 |
+
return donors
|
| 342 |
+
|
| 343 |
+
|
| 344 |
+
def cluster_bootstrap_interval(
|
| 345 |
+
values: np.ndarray,
|
| 346 |
+
recording: np.ndarray,
|
| 347 |
+
statistic,
|
| 348 |
+
*,
|
| 349 |
+
samples: int = 2000,
|
| 350 |
+
seed: int = 20260911,
|
| 351 |
+
) -> tuple[float, float]:
|
| 352 |
+
"""Bootstrap complete recordings and return a percentile interval."""
|
| 353 |
+
|
| 354 |
+
recordings = np.unique(recording)
|
| 355 |
+
groups = {value: np.flatnonzero(recording == value) for value in recordings}
|
| 356 |
+
rng = np.random.default_rng(seed)
|
| 357 |
+
estimates = np.empty(samples, dtype=np.float64)
|
| 358 |
+
for draw in range(samples):
|
| 359 |
+
selected = rng.choice(recordings, size=len(recordings), replace=True)
|
| 360 |
+
indices = np.concatenate([groups[value] for value in selected])
|
| 361 |
+
estimates[draw] = float(statistic(values, indices))
|
| 362 |
+
return tuple(np.quantile(estimates, [0.025, 0.975]).tolist())
|
model/tachin_paired30_cache_20260923/code/tests/test_tachin_native_editor.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
|
| 3 |
+
from tactile_editor.native_tachin import (NativeResidualEditor, NativeTachinEncoder,
|
| 4 |
+
NativeTachinW3Editor, SENSOR_OFFSETS,
|
| 5 |
+
SENSOR_SHAPES)
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def inputs(batch=2):
|
| 9 |
+
mean = torch.linspace(0, 0.2, 1760).reshape(2, 880)
|
| 10 |
+
scale = torch.ones(2, 880) * 0.1
|
| 11 |
+
pressure = mean[None, None].expand(batch, 8, -1, -1).clone()
|
| 12 |
+
cell_valid = torch.ones_like(pressure, dtype=torch.bool)
|
| 13 |
+
sample_valid = torch.ones(batch, 8, 2, dtype=torch.bool)
|
| 14 |
+
return mean, scale, pressure, cell_valid, sample_valid
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def test_native_regions_cover_each_cell_once():
|
| 18 |
+
assert SENSOR_OFFSETS[0] == 0 and SENSOR_OFFSETS[-1] == 880
|
| 19 |
+
assert all(stop - start == height * width for (height, width), start, stop
|
| 20 |
+
in zip(SENSOR_SHAPES, SENSOR_OFFSETS[:-1], SENSOR_OFFSETS[1:]))
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def test_reference_touch_is_exact_base_for_memory_free_editor():
|
| 24 |
+
mean, scale, pressure, cell_valid, sample_valid = inputs()
|
| 25 |
+
model = NativeResidualEditor(NativeTachinEncoder(mean, scale)).eval()
|
| 26 |
+
base = torch.randn(2, 16, 192)
|
| 27 |
+
mask = torch.ones_like(base, dtype=torch.bool)
|
| 28 |
+
final, delta, allowed, features, hand_valid = model(
|
| 29 |
+
base, mask, pressure, cell_valid, sample_valid)
|
| 30 |
+
assert torch.equal(final, base)
|
| 31 |
+
assert not delta.bool().any()
|
| 32 |
+
assert not features.bool().any()
|
| 33 |
+
assert allowed[..., :138].all() and not allowed[..., 138:].any()
|
| 34 |
+
assert hand_valid.all()
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def test_reference_touch_stays_exact_after_coefficient_bias_is_trained():
|
| 38 |
+
mean, scale, pressure, cell_valid, sample_valid = inputs()
|
| 39 |
+
pressure = mean[None, None].expand_as(pressure).clone()
|
| 40 |
+
model = NativeResidualEditor(NativeTachinEncoder(mean, scale)).eval()
|
| 41 |
+
with torch.no_grad():
|
| 42 |
+
model.coefficients[0].bias.fill_(0.5)
|
| 43 |
+
model.action.basis_head.bias.fill_(0.5)
|
| 44 |
+
base = torch.randn(2, 16, 192)
|
| 45 |
+
final, delta, _, features, _ = model(
|
| 46 |
+
base, torch.ones_like(base, dtype=torch.bool), pressure,
|
| 47 |
+
cell_valid, sample_valid,
|
| 48 |
+
)
|
| 49 |
+
assert not features.bool().any()
|
| 50 |
+
assert not delta.bool().any()
|
| 51 |
+
assert torch.equal(final, base)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def test_native_w3_no_auxiliary_shape_and_reference_invariance():
|
| 55 |
+
mean, scale, pressure, cell_valid, sample_valid = inputs(batch=1)
|
| 56 |
+
model = NativeTachinW3Editor(NativeTachinEncoder(mean, scale), mean, scale).eval()
|
| 57 |
+
base = torch.randn(1, 16, 192)
|
| 58 |
+
output = model(base_action=base, edit_mask=torch.ones_like(base, dtype=torch.bool),
|
| 59 |
+
pressure=pressure, cell_valid=cell_valid,
|
| 60 |
+
sample_valid=sample_valid, final_memory=torch.randn(1, 8, 960),
|
| 61 |
+
compute_auxiliary=False)
|
| 62 |
+
assert torch.equal(output.final_action, base)
|
| 63 |
+
assert output.tactile_velocity is None and output.rgb is None
|
model/tachin_paired30_cache_20260923/environment.txt
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
python==3.12.3
|
| 2 |
+
torch==2.8.0+cu128
|
| 3 |
+
numpy==1.26.4
|
| 4 |
+
scipy==1.17.1
|
| 5 |
+
huggingface_hub==1.32.0
|
| 6 |
+
transformers==5.17.0
|
| 7 |
+
|
model/twla_tachin_common_adaptation_20260922/README.md
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# T-WLA Tachin common-target adaptation
|
| 2 |
+
|
| 3 |
+
This is the T-WLA checkpoint used to generate the Tachin paired-30 W3 cache.
|
| 4 |
+
No MemoryVLA checkpoint, cache, or adaptation result is included in this
|
| 5 |
+
release.
|
| 6 |
+
|
| 7 |
+
## Adaptation contract
|
| 8 |
+
|
| 9 |
+
- Initialization: T-WLA ten-domain 110k checkpoint
|
| 10 |
+
(`v4-runs:/v4d_ten_stageA/best.pt`).
|
| 11 |
+
- Initial checkpoint SHA-256:
|
| 12 |
+
`ccd65fcf845ec01c7de46bb76e9cf0fee33674dd848771e114ff23b2535f13d9`.
|
| 13 |
+
- Dataset: Tachin training/validation split used by all Table 1 methods.
|
| 14 |
+
- Targets and action masks: the exact common Tachin implementation shared by
|
| 15 |
+
the baseline adaptations.
|
| 16 |
+
- Training: 1,000 optimizer steps, effective batch size 128, learning rate
|
| 17 |
+
`5e-5`, and 100 warm-up steps.
|
| 18 |
+
- Frozen: VLM and visual-memory modules.
|
| 19 |
+
- Trainable: action expert only (102,968,832 parameters).
|
| 20 |
+
- Checkpoint selection: validation loss only; the test split was not used.
|
| 21 |
+
- Selected step: 800.
|
| 22 |
+
- Validation loss: 0.79419 before adaptation and 0.55530 after adaptation.
|
| 23 |
+
|
| 24 |
+
`checkpoint/best.pt` is the complete, directly loadable checkpoint. It was
|
| 25 |
+
assembled from the original frozen tensors, the adapted action expert, and
|
| 26 |
+
Tachin train-only normalization statistics. Its SHA-256 is
|
| 27 |
+
`8df6c87fda2e5ec09d61686451744c37bdcc2d1b4d9d13b58893851be620db7d`.
|
| 28 |
+
|
| 29 |
+
`checkpoint/best_adapter.pt` contains the selected expert-only adapter. The
|
| 30 |
+
script `code/rebuild_twla_adapted_checkpoint.py` can merge it with the original
|
| 31 |
+
ten-domain checkpoint. The complete `best.pt` does not require the original
|
| 32 |
+
checkpoint for inference.
|
| 33 |
+
|
| 34 |
+
The verifier confirmed that all 381 VLM/memory tensors are bit-identical to the
|
| 35 |
+
initial checkpoint and that 155 expert tensors changed. See
|
| 36 |
+
`provenance/verification.json` and `SHA256SUMS`.
|
| 37 |
+
|
| 38 |
+
To retrieve the original initialization from Modal, use:
|
| 39 |
+
|
| 40 |
+
```bash
|
| 41 |
+
modal volume get v4-runs /v4d_ten_stageA/best.pt
|
| 42 |
+
```
|
| 43 |
+
|
| 44 |
+
The paired-30 policy cache and tactile-editor artifacts generated from this
|
| 45 |
+
checkpoint are published in `model/tachin_paired30_cache_20260923`.
|
model/twla_tachin_common_adaptation_20260922/SHA256SUMS
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
35a777f79038f0e009be3da6978b75a42202f7f3ce842cbb3b442fca5efe6411 ./README.md
|
| 2 |
+
8df6c87fda2e5ec09d61686451744c37bdcc2d1b4d9d13b58893851be620db7d ./checkpoint/best.pt
|
| 3 |
+
caeb8b59c909728941b49af3237e1639f75a26a11a30d487dd4d1cff990b4050 ./checkpoint/best_adapter.pt
|
| 4 |
+
3515348b90c53e42d2eb4f7435a6c922a5a89d097126a12eda10335ce5c7c7c5 ./code/adapt_twla_tachin_common.py
|
| 5 |
+
f89146ac7f47157deea98e6dd713093f5c80d25af811a8581f553e593de9e7ea ./code/cache_tachin_twla_w3.py
|
| 6 |
+
0196f9a4cb9fa7b5758501529efb7910a07aafd14c80ef6831578b86273fdd05 ./code/rebuild_twla_adapted_checkpoint.py
|
| 7 |
+
12912c259f733d3fedf91a567c94c18a2573a4b1a33c428fb9fe04b9dabd9602 ./code/verify_twla_tachin_adaptation.py
|
| 8 |
+
6f6263f2b8be7f7fe7a55296488e1888ac16aaa2fa37125a327c2acf29686bc9 ./provenance/COMPLETE.json
|
| 9 |
+
bd7ea64950e76084e820f1c2cc0d2700115599214e137103f22e4bcf6f5f1c29 ./provenance/config.json
|
| 10 |
+
140ec19765ec0752f9af2dd8918d1fda23e5c436392fc15ca9ba25f41fbf8d7a ./provenance/initial_config_used.json
|
| 11 |
+
140ec19765ec0752f9af2dd8918d1fda23e5c436392fc15ca9ba25f41fbf8d7a ./provenance/initial_ten_domain_config.json
|
| 12 |
+
4ae9e15ca2706fb1160af13ca6a858c12efc11ff64a7ee237f07cfd8d02a212b ./provenance/metrics.jsonl
|
| 13 |
+
dfb99ee046015d443c068e21e3f58fa0d882c22be6eba2384269b5b308714994 ./provenance/verification.json
|
model/twla_tachin_common_adaptation_20260922/code/adapt_twla_tachin_common.py
ADDED
|
@@ -0,0 +1,374 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Adapt T-WLA-110k to Tachin using the paper's exact common action targets.
|
| 2 |
+
|
| 3 |
+
The VLM and recurrent memory remain frozen. Only the action expert is adapted, which
|
| 4 |
+
matches the expert-only baseline protocol and keeps the frozen RGB world head compatible
|
| 5 |
+
with the 110k memory representation. Inputs use the same Tachin episode split and anchor
|
| 6 |
+
rows as the common Table-1 cache; labels and masks are read directly from that cache.
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import argparse
|
| 11 |
+
import hashlib
|
| 12 |
+
import json
|
| 13 |
+
import math
|
| 14 |
+
import os
|
| 15 |
+
import random
|
| 16 |
+
import shutil
|
| 17 |
+
import sys
|
| 18 |
+
import time
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
|
| 21 |
+
import numpy as np
|
| 22 |
+
import torch
|
| 23 |
+
from torch.utils.data import DataLoader, Dataset
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 27 |
+
V4 = ROOT / "ref/yuzhench_distill_editor/V4"
|
| 28 |
+
sys.path[:0] = [str(ROOT / "src"), str(V4), str(V4 / "vla_hand_pose"),
|
| 29 |
+
str(ROOT / "scripts")]
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def digest(path: Path) -> str:
|
| 33 |
+
value = hashlib.sha256()
|
| 34 |
+
with path.open("rb") as stream:
|
| 35 |
+
for block in iter(lambda: stream.read(8 << 20), b""):
|
| 36 |
+
value.update(block)
|
| 37 |
+
return value.hexdigest()
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def atomic_torch(payload, path: Path) -> None:
|
| 41 |
+
temporary = path.with_suffix(path.suffix + ".tmp")
|
| 42 |
+
torch.save(payload, temporary)
|
| 43 |
+
os.replace(temporary, path)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def atomic_json(payload, path: Path) -> None:
|
| 47 |
+
temporary = path.with_suffix(path.suffix + ".tmp")
|
| 48 |
+
temporary.write_text(json.dumps(payload, indent=2, default=str) + "\n")
|
| 49 |
+
os.replace(temporary, path)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
class IndexedPlan(Dataset):
|
| 53 |
+
def __init__(self, base, indices):
|
| 54 |
+
self.base = base
|
| 55 |
+
self.indices = np.asarray(indices, dtype=np.int64)
|
| 56 |
+
|
| 57 |
+
def __len__(self):
|
| 58 |
+
return len(self.indices)
|
| 59 |
+
|
| 60 |
+
def __getitem__(self, index):
|
| 61 |
+
item = self.base[index]
|
| 62 |
+
item["query_index"] = int(self.indices[index])
|
| 63 |
+
return item
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def planned_dataset(split: str, args, config, tokenizer, source_cls, dataset_cls):
|
| 67 |
+
rows = json.loads((args.adaptation_data / split / "rows.json").read_text())
|
| 68 |
+
identity = dict(np.load(args.common / "shared" / split / "identity.npz",
|
| 69 |
+
allow_pickle=False))
|
| 70 |
+
row_ids = np.asarray([row["sample_id"] for row in rows]).astype(str)
|
| 71 |
+
if not np.array_equal(row_ids, identity["sample_id"].astype(str)):
|
| 72 |
+
raise ValueError(f"{split} prepared rows do not match the common cache")
|
| 73 |
+
source = source_cls(
|
| 74 |
+
name="tachin", video_root="/unused",
|
| 75 |
+
annotation_file=str(args.annotations / "episode_frame_index.npz"),
|
| 76 |
+
label_folder=str(args.annotations / "episodic_annotations"),
|
| 77 |
+
jpeg_root=str(args.jpeg_root),
|
| 78 |
+
)
|
| 79 |
+
episodes = {str(row["episode_id"]) for row in rows}
|
| 80 |
+
dataset = dataset_cls([source], cfg=config,
|
| 81 |
+
video_whitelist={"tachin": episodes}, tokenizer=tokenizer,
|
| 82 |
+
seed=args.seed, load_images=True, training=False,
|
| 83 |
+
deterministic=True)
|
| 84 |
+
units = {(unit.episode, unit.phase): index
|
| 85 |
+
for index, unit in enumerate(dataset.units)}
|
| 86 |
+
plans = []
|
| 87 |
+
for common_index, row in enumerate(rows):
|
| 88 |
+
anchor = int(row["anchor_source_row"])
|
| 89 |
+
phase, relative = anchor % 3, anchor // 3
|
| 90 |
+
length = min(relative + 1, 9)
|
| 91 |
+
key = (str(row["episode_id"]), phase)
|
| 92 |
+
if key not in units:
|
| 93 |
+
raise KeyError(f"Missing T-WLA sequence unit {key}")
|
| 94 |
+
plans.append((units[key], relative - length + 1, length, common_index))
|
| 95 |
+
dataset.set_eval_plan(plans)
|
| 96 |
+
return IndexedPlan(dataset, np.arange(len(rows))), dataset
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def set_train_mode(model) -> None:
|
| 100 |
+
model.eval()
|
| 101 |
+
model.expert.train()
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def common_batch(batch, split: str, targets, masks, device, observation_cls,
|
| 105 |
+
to_observations):
|
| 106 |
+
steps, step_valid, _, _ = to_observations(
|
| 107 |
+
batch, device, observation_cls, dtype=torch.bfloat16)
|
| 108 |
+
indices = batch["query_index"].numpy()
|
| 109 |
+
actions = torch.from_numpy(np.asarray(targets[split][indices]).copy()).to(
|
| 110 |
+
device, dtype=torch.float32)
|
| 111 |
+
action_mask = torch.from_numpy(np.asarray(masks[split][indices]).copy()).to(
|
| 112 |
+
device, dtype=torch.bool)
|
| 113 |
+
return steps, step_valid, actions, action_mask
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
@torch.no_grad()
|
| 117 |
+
def validate(model, loader, split, targets, masks, device, observation_cls,
|
| 118 |
+
to_observations, seed):
|
| 119 |
+
model.eval()
|
| 120 |
+
total = 0.0
|
| 121 |
+
count = 0
|
| 122 |
+
generator = torch.Generator(device=device).manual_seed(seed)
|
| 123 |
+
for batch in loader:
|
| 124 |
+
values = common_batch(batch, split, targets, masks, device, observation_cls,
|
| 125 |
+
to_observations)
|
| 126 |
+
with torch.autocast("cuda", dtype=torch.bfloat16):
|
| 127 |
+
loss, _ = model.loss(*values, generator=generator)
|
| 128 |
+
if not torch.isfinite(loss):
|
| 129 |
+
raise ValueError("Nonfinite T-WLA validation loss")
|
| 130 |
+
size = len(batch["query_index"])
|
| 131 |
+
total += float(loss) * size
|
| 132 |
+
count += size
|
| 133 |
+
return total / max(count, 1)
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def main() -> None:
|
| 137 |
+
parser = argparse.ArgumentParser()
|
| 138 |
+
parser.add_argument("--checkpoint", type=Path, required=True)
|
| 139 |
+
parser.add_argument("--checkpoint-config", type=Path, required=True)
|
| 140 |
+
parser.add_argument("--annotations", type=Path, required=True)
|
| 141 |
+
parser.add_argument("--jpeg-root", type=Path, required=True)
|
| 142 |
+
parser.add_argument("--adaptation-data", type=Path, required=True)
|
| 143 |
+
parser.add_argument("--common", type=Path, required=True)
|
| 144 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 145 |
+
parser.add_argument("--steps", type=int, default=1000)
|
| 146 |
+
parser.add_argument("--batch-size", type=int, default=32)
|
| 147 |
+
parser.add_argument("--accumulation", type=int, default=4)
|
| 148 |
+
parser.add_argument("--lr", type=float, default=5e-5)
|
| 149 |
+
parser.add_argument("--warmup", type=int, default=100)
|
| 150 |
+
parser.add_argument("--val-every", type=int, default=100)
|
| 151 |
+
parser.add_argument("--workers", type=int, default=2)
|
| 152 |
+
parser.add_argument("--seed", type=int, default=20260922)
|
| 153 |
+
parser.add_argument("--smoke", action="store_true")
|
| 154 |
+
args = parser.parse_args()
|
| 155 |
+
if min(args.steps, args.batch_size, args.accumulation, args.val_every) < 1:
|
| 156 |
+
raise ValueError("steps, batch-size, accumulation, and val-every must be positive")
|
| 157 |
+
if args.lr <= 0 or args.warmup < 0 or args.workers < 0:
|
| 158 |
+
raise ValueError("invalid optimizer or loader setting")
|
| 159 |
+
if args.output.exists():
|
| 160 |
+
raise FileExistsError(args.output)
|
| 161 |
+
if args.smoke:
|
| 162 |
+
args.steps = 2
|
| 163 |
+
args.batch_size = 2
|
| 164 |
+
args.accumulation = 1
|
| 165 |
+
args.val_every = 2
|
| 166 |
+
args.warmup = 0
|
| 167 |
+
args.output.mkdir(parents=True)
|
| 168 |
+
|
| 169 |
+
os.environ.update(
|
| 170 |
+
METHOD_D_CHECKPOINT=str(args.checkpoint.resolve()),
|
| 171 |
+
METHOD_D_RUN_CONFIG=str(args.checkpoint_config.resolve()),
|
| 172 |
+
METHOD_D_SOURCE=str(V4 / "vla_hand_pose"),
|
| 173 |
+
)
|
| 174 |
+
from transformers import AutoTokenizer
|
| 175 |
+
from modal_build_full_editor_cache import load_backbone
|
| 176 |
+
from vla_hand_pose.dataloader import (SequenceDataset, Source, make_collate,
|
| 177 |
+
to_observations)
|
| 178 |
+
from vla_hand_pose.v4_d_fm_0 import Observation
|
| 179 |
+
|
| 180 |
+
random.seed(args.seed)
|
| 181 |
+
np.random.seed(args.seed)
|
| 182 |
+
torch.manual_seed(args.seed)
|
| 183 |
+
torch.cuda.manual_seed_all(args.seed)
|
| 184 |
+
torch.set_num_threads(4)
|
| 185 |
+
torch.set_float32_matmul_precision("high")
|
| 186 |
+
device = torch.device("cuda:0")
|
| 187 |
+
torch.cuda.set_device(device)
|
| 188 |
+
|
| 189 |
+
model, config = load_backbone(device)
|
| 190 |
+
statistics = json.loads((args.adaptation_data / "statistics.json").read_text())[
|
| 191 |
+
"keypoints"]
|
| 192 |
+
model.set_normalisation(**{name: statistics[name] for name in
|
| 193 |
+
("action_mean", "action_std", "state_mean", "state_std")})
|
| 194 |
+
model.requires_grad_(False)
|
| 195 |
+
model.expert.requires_grad_(True)
|
| 196 |
+
trainable = [parameter for parameter in model.expert.parameters()
|
| 197 |
+
if parameter.requires_grad]
|
| 198 |
+
tokenizer = AutoTokenizer.from_pretrained(config.vlm.model_id)
|
| 199 |
+
|
| 200 |
+
targets = {}
|
| 201 |
+
masks = {}
|
| 202 |
+
loaders = {}
|
| 203 |
+
split_rows = {}
|
| 204 |
+
for split in ("train", "val"):
|
| 205 |
+
targets[split] = np.load(args.common / "shared" / split /
|
| 206 |
+
"target_action.npy", mmap_mode="r")
|
| 207 |
+
masks[split] = np.load(args.common / "shared" / split /
|
| 208 |
+
"action_mask.npy", mmap_mode="r")
|
| 209 |
+
wrapped, base = planned_dataset(split, args, config, tokenizer, Source,
|
| 210 |
+
SequenceDataset)
|
| 211 |
+
base_collate = make_collate(config, tokenizer)
|
| 212 |
+
|
| 213 |
+
def collate(items, base_collate=base_collate):
|
| 214 |
+
batch = base_collate(items)
|
| 215 |
+
batch["query_index"] = torch.as_tensor(
|
| 216 |
+
[item["query_index"] for item in items], dtype=torch.int64)
|
| 217 |
+
return batch
|
| 218 |
+
|
| 219 |
+
loaders[split] = DataLoader(
|
| 220 |
+
wrapped, batch_size=args.batch_size,
|
| 221 |
+
shuffle=split == "train",
|
| 222 |
+
generator=torch.Generator().manual_seed(args.seed),
|
| 223 |
+
num_workers=args.workers, collate_fn=collate, pin_memory=True,
|
| 224 |
+
persistent_workers=args.workers > 0,
|
| 225 |
+
)
|
| 226 |
+
split_rows[split] = len(wrapped)
|
| 227 |
+
|
| 228 |
+
groups = model.param_groups()
|
| 229 |
+
if {id(parameter) for group in groups for parameter in group["params"]} != {
|
| 230 |
+
id(parameter) for parameter in trainable}:
|
| 231 |
+
raise ValueError("optimizer groups do not exactly cover the action expert")
|
| 232 |
+
for group in groups:
|
| 233 |
+
group["lr"] = args.lr
|
| 234 |
+
group["base_lr"] = args.lr
|
| 235 |
+
group["weight_decay"] = 0.01 if group["name"] == "decay" else 0.0
|
| 236 |
+
optimizer = torch.optim.AdamW(groups, lr=args.lr, betas=(0.9, 0.95), eps=1e-8)
|
| 237 |
+
|
| 238 |
+
run_config = {
|
| 239 |
+
**{name: str(value) if isinstance(value, Path) else value
|
| 240 |
+
for name, value in vars(args).items()},
|
| 241 |
+
"protocol": "Tachin-only expert adaptation from T-WLA-110k",
|
| 242 |
+
"targets": "exact common cache target_action/action_mask",
|
| 243 |
+
"normalization": "exact common Tachin train-only keypoint statistics",
|
| 244 |
+
"history": "up to eight prior 10-Hz observations plus current",
|
| 245 |
+
"frozen": ["vlm", "memory"],
|
| 246 |
+
"trainable": ["expert"],
|
| 247 |
+
"train_rows": split_rows["train"],
|
| 248 |
+
"val_rows": split_rows["val"],
|
| 249 |
+
"initial_sha256": digest(args.checkpoint),
|
| 250 |
+
"common_complete_sha256": digest(args.common / "complete.json"),
|
| 251 |
+
}
|
| 252 |
+
atomic_json(run_config, args.output / "config.json")
|
| 253 |
+
shutil.copy2(args.checkpoint_config, args.output / "initial_config_used.json")
|
| 254 |
+
|
| 255 |
+
generator = torch.Generator(device=device).manual_seed(args.seed + 1)
|
| 256 |
+
best = validate(model, loaders["val"], "val", targets, masks, device,
|
| 257 |
+
Observation, to_observations, args.seed + 20000)
|
| 258 |
+
best_step = 0
|
| 259 |
+
atomic_torch({"expert": model.expert.state_dict(), "step": 0,
|
| 260 |
+
"val_loss": best}, args.output / "best_adapter.pt")
|
| 261 |
+
started = time.time()
|
| 262 |
+
samples_seen = 0
|
| 263 |
+
iterator = iter(loaders["train"])
|
| 264 |
+
with (args.output / "metrics.jsonl").open("a") as stream:
|
| 265 |
+
stream.write(json.dumps({"step": 0, "val_loss": best, "seconds": 0.0}) + "\n")
|
| 266 |
+
print(json.dumps({"step": 0, "val_loss": best}), flush=True)
|
| 267 |
+
|
| 268 |
+
for step in range(1, args.steps + 1):
|
| 269 |
+
set_train_mode(model)
|
| 270 |
+
optimizer.zero_grad(set_to_none=True)
|
| 271 |
+
train_loss = 0.0
|
| 272 |
+
step_samples = 0
|
| 273 |
+
if args.warmup and step <= args.warmup:
|
| 274 |
+
factor = step / args.warmup
|
| 275 |
+
else:
|
| 276 |
+
factor = 0.1 + 0.45 * (1 + math.cos(
|
| 277 |
+
math.pi * (step - args.warmup) / max(1, args.steps - args.warmup)))
|
| 278 |
+
for group in optimizer.param_groups:
|
| 279 |
+
group["lr"] = group["base_lr"] * factor
|
| 280 |
+
for _ in range(args.accumulation):
|
| 281 |
+
try:
|
| 282 |
+
batch = next(iterator)
|
| 283 |
+
except StopIteration:
|
| 284 |
+
iterator = iter(loaders["train"])
|
| 285 |
+
batch = next(iterator)
|
| 286 |
+
values = common_batch(batch, "train", targets, masks, device,
|
| 287 |
+
Observation, to_observations)
|
| 288 |
+
with torch.autocast("cuda", dtype=torch.bfloat16):
|
| 289 |
+
loss, _ = model.loss(*values, generator=generator)
|
| 290 |
+
scaled = loss / args.accumulation
|
| 291 |
+
if not torch.isfinite(scaled):
|
| 292 |
+
raise ValueError("Nonfinite T-WLA adaptation loss")
|
| 293 |
+
scaled.backward()
|
| 294 |
+
train_loss += float(scaled.detach())
|
| 295 |
+
step_samples += len(batch["query_index"])
|
| 296 |
+
gradient = torch.nn.utils.clip_grad_norm_(trainable, 1.0,
|
| 297 |
+
error_if_nonfinite=True)
|
| 298 |
+
optimizer.step()
|
| 299 |
+
samples_seen += step_samples
|
| 300 |
+
record = {
|
| 301 |
+
"step": step, "train_loss": train_loss,
|
| 302 |
+
"grad_norm": float(gradient), "samples_seen": samples_seen,
|
| 303 |
+
"equivalent_epochs": samples_seen / split_rows["train"],
|
| 304 |
+
"learning_rates": [group["lr"] for group in optimizer.param_groups],
|
| 305 |
+
"seconds": time.time() - started,
|
| 306 |
+
"peak_gpu_gb": torch.cuda.max_memory_allocated() / 1e9,
|
| 307 |
+
}
|
| 308 |
+
if step % args.val_every == 0 or step == args.steps:
|
| 309 |
+
value = validate(model, loaders["val"], "val", targets, masks, device,
|
| 310 |
+
Observation, to_observations, args.seed + 20000)
|
| 311 |
+
record["val_loss"] = value
|
| 312 |
+
if value < best:
|
| 313 |
+
best = value
|
| 314 |
+
best_step = step
|
| 315 |
+
atomic_torch({"expert": model.expert.state_dict(), "step": step,
|
| 316 |
+
"val_loss": value}, args.output / "best_adapter.pt")
|
| 317 |
+
atomic_torch({
|
| 318 |
+
"expert": model.expert.state_dict(), "optimizer": optimizer.state_dict(),
|
| 319 |
+
"step": step, "best": best, "best_step": best_step,
|
| 320 |
+
"generator_state": generator.get_state(),
|
| 321 |
+
}, args.output / "last_adapter.pt")
|
| 322 |
+
with (args.output / "metrics.jsonl").open("a") as stream:
|
| 323 |
+
stream.write(json.dumps(record) + "\n")
|
| 324 |
+
if step == 1 or step % 10 == 0 or "val_loss" in record:
|
| 325 |
+
print(json.dumps(record), flush=True)
|
| 326 |
+
|
| 327 |
+
selected = torch.load(args.output / "best_adapter.pt", map_location="cpu",
|
| 328 |
+
weights_only=False)
|
| 329 |
+
# Do not serialize the complete live model here. ``load_backbone`` casts it to
|
| 330 |
+
# bfloat16, so doing that would numerically change every frozen FP32 tensor even
|
| 331 |
+
# though only the expert was optimized. Assemble from the source checkpoint and
|
| 332 |
+
# replace only the expert and Tachin normalization buffers.
|
| 333 |
+
source = torch.load(args.checkpoint, map_location="cpu", mmap=True,
|
| 334 |
+
weights_only=False)
|
| 335 |
+
source_state = dict(source["model"])
|
| 336 |
+
for short_name, value in selected["expert"].items():
|
| 337 |
+
name = f"expert.{short_name}"
|
| 338 |
+
if name not in source_state:
|
| 339 |
+
raise KeyError(f"Adapted expert tensor is absent from source checkpoint: {name}")
|
| 340 |
+
source_state[name] = value.to(dtype=source_state[name].dtype)
|
| 341 |
+
for name in ("action_mean", "action_std", "state_mean", "state_std"):
|
| 342 |
+
source_state[name] = torch.as_tensor(
|
| 343 |
+
np.asarray(statistics[name]), dtype=source_state[name].dtype)
|
| 344 |
+
source_state["stats_loaded"] = torch.ones_like(
|
| 345 |
+
source_state["stats_loaded"], dtype=torch.bool)
|
| 346 |
+
full = {
|
| 347 |
+
"model": source_state, "step": int(selected["step"]), "epoch": -1,
|
| 348 |
+
"best_val": float(selected["val_loss"]),
|
| 349 |
+
"adaptation": {
|
| 350 |
+
**run_config,
|
| 351 |
+
"assembly": "original frozen state + adapted expert + Tachin normalization",
|
| 352 |
+
"frozen_tensor_contract": "vlm.* and memory.* bit-identical to initial",
|
| 353 |
+
},
|
| 354 |
+
}
|
| 355 |
+
atomic_torch(full, args.output / "best.pt")
|
| 356 |
+
atomic_json({
|
| 357 |
+
"schema": "twla.tachin_common_adaptation.v1",
|
| 358 |
+
"selected_step": int(selected["step"]),
|
| 359 |
+
"best_validation_loss": float(selected["val_loss"]),
|
| 360 |
+
"initial_checkpoint": str(args.checkpoint.resolve()),
|
| 361 |
+
"initial_sha256": run_config["initial_sha256"],
|
| 362 |
+
"checkpoint_sha256": digest(args.output / "best.pt"),
|
| 363 |
+
"test_used": False,
|
| 364 |
+
"trainable_parameters": sum(parameter.numel() for parameter in trainable),
|
| 365 |
+
"frozen_parameters": sum(parameter.numel() for parameter in model.parameters()
|
| 366 |
+
if not parameter.requires_grad),
|
| 367 |
+
"seconds": time.time() - started,
|
| 368 |
+
}, args.output / "COMPLETE.json")
|
| 369 |
+
print(json.dumps({"status": "complete", "selected_step": int(selected["step"]),
|
| 370 |
+
"best_validation_loss": float(selected["val_loss"])}), flush=True)
|
| 371 |
+
|
| 372 |
+
|
| 373 |
+
if __name__ == "__main__":
|
| 374 |
+
main()
|
model/twla_tachin_common_adaptation_20260922/code/cache_tachin_twla_w3.py
ADDED
|
@@ -0,0 +1,299 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Cache T-WLA-110k actions, recurrent memory, and W3 RGB-world inputs."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import argparse
|
| 5 |
+
import copy
|
| 6 |
+
import hashlib
|
| 7 |
+
import json
|
| 8 |
+
import os
|
| 9 |
+
import sys
|
| 10 |
+
import time
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
|
| 13 |
+
import numpy as np
|
| 14 |
+
import torch
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 18 |
+
V4 = ROOT / "ref/yuzhench_distill_editor/V4"
|
| 19 |
+
sys.path[:0] = [str(ROOT / "src"), str(ROOT / "ref/Tactile"), str(V4), str(V4 / "vla_hand_pose"), str(ROOT / "scripts")]
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def sha256(path: Path) -> str:
|
| 23 |
+
h = hashlib.sha256()
|
| 24 |
+
with path.open("rb") as stream:
|
| 25 |
+
for block in iter(lambda: stream.read(8 << 20), b""):
|
| 26 |
+
h.update(block)
|
| 27 |
+
return h.hexdigest()
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def array(root: Path, name: str, shape, dtype):
|
| 31 |
+
root.mkdir(parents=True, exist_ok=True)
|
| 32 |
+
path = root / f"{name}.npy"
|
| 33 |
+
if path.exists():
|
| 34 |
+
value = np.load(path, mmap_mode="r+")
|
| 35 |
+
if value.shape != tuple(shape) or value.dtype != np.dtype(dtype):
|
| 36 |
+
raise ValueError(f"Existing cache schema mismatch: {path}")
|
| 37 |
+
return value
|
| 38 |
+
return np.lib.format.open_memmap(path, mode="w+", shape=shape, dtype=dtype)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def atomic_json(path: Path, payload) -> None:
|
| 42 |
+
temporary = path.with_suffix(path.suffix + ".tmp")
|
| 43 |
+
temporary.write_text(json.dumps(payload, indent=2, default=str) + "\n")
|
| 44 |
+
os.replace(temporary, path)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def checkpoint_provenance(path: Path) -> dict:
|
| 48 |
+
"""Read cache-facing provenance without assuming the 110k initialization."""
|
| 49 |
+
payload = torch.load(path, map_location="cpu", mmap=True, weights_only=False)
|
| 50 |
+
step = payload.get("optimizer_step", payload.get("step"))
|
| 51 |
+
if step is None:
|
| 52 |
+
raise ValueError(f"Checkpoint does not declare a training/adaptation step: {path}")
|
| 53 |
+
adaptation = payload.get("adaptation")
|
| 54 |
+
result = {
|
| 55 |
+
"checkpoint_step": int(step),
|
| 56 |
+
"checkpoint_is_tachin_adapted": isinstance(adaptation, dict),
|
| 57 |
+
}
|
| 58 |
+
if isinstance(adaptation, dict):
|
| 59 |
+
result.update(
|
| 60 |
+
adaptation_protocol=adaptation.get("protocol"),
|
| 61 |
+
adaptation_initial_checkpoint=adaptation.get("checkpoint"),
|
| 62 |
+
adaptation_initial_sha256=adaptation.get("initial_sha256"),
|
| 63 |
+
adaptation_common_complete_sha256=adaptation.get("common_complete_sha256"),
|
| 64 |
+
)
|
| 65 |
+
del payload
|
| 66 |
+
return result
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
class PlannedDataset:
|
| 70 |
+
def __init__(self, base, common_indices, anchors):
|
| 71 |
+
self.base = base
|
| 72 |
+
self.common_indices = np.asarray(common_indices)
|
| 73 |
+
self.anchors = np.asarray(anchors)
|
| 74 |
+
|
| 75 |
+
def __len__(self):
|
| 76 |
+
return len(self.common_indices)
|
| 77 |
+
|
| 78 |
+
def __getitem__(self, index):
|
| 79 |
+
from tactile_editor.rgb_world_data import project_future_hand_roi
|
| 80 |
+
item = self.base[index]
|
| 81 |
+
unit = self.base.units[self.base._eval_plan[index][0]]
|
| 82 |
+
annotation, _, _ = self.base.cores[unit.source]._load_or_cache_episode(unit.episode)
|
| 83 |
+
anchor = int(self.anchors[index])
|
| 84 |
+
roi = np.zeros((2, 16, 16), bool)
|
| 85 |
+
roi_valid = np.zeros(2, bool)
|
| 86 |
+
hand = annotation.get(str(annotation["anno_type"]))
|
| 87 |
+
if isinstance(hand, dict):
|
| 88 |
+
for future_index, offset in enumerate((3, 6)):
|
| 89 |
+
if anchor + offset < len(annotation["video_decode_frame"]):
|
| 90 |
+
projected = project_future_hand_roi(
|
| 91 |
+
np.asarray(hand["joints_camspace"])[anchor + offset],
|
| 92 |
+
annotation["intrinsics"], source_hw=(1080, 1920),
|
| 93 |
+
)
|
| 94 |
+
roi[future_index] = projected.roi
|
| 95 |
+
roi_valid[future_index] = projected.valid
|
| 96 |
+
item.update(query_index=int(self.common_indices[index]), future_roi=roi,
|
| 97 |
+
future_roi_valid=roi_valid)
|
| 98 |
+
return item
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def main() -> None:
|
| 102 |
+
parser = argparse.ArgumentParser()
|
| 103 |
+
parser.add_argument("--checkpoint", type=Path, required=True)
|
| 104 |
+
parser.add_argument("--checkpoint-config", type=Path, required=True)
|
| 105 |
+
parser.add_argument("--world-checkpoint", type=Path, required=True)
|
| 106 |
+
parser.add_argument("--world-config", type=Path, required=True)
|
| 107 |
+
parser.add_argument("--annotations", type=Path, required=True)
|
| 108 |
+
parser.add_argument("--jpeg-root", type=Path, required=True)
|
| 109 |
+
parser.add_argument("--adaptation-data", type=Path, required=True)
|
| 110 |
+
parser.add_argument("--common", type=Path, required=True)
|
| 111 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 112 |
+
parser.add_argument("--shard", type=int, required=True)
|
| 113 |
+
parser.add_argument("--shards", type=int, default=4)
|
| 114 |
+
parser.add_argument("--batch-size", type=int, default=24)
|
| 115 |
+
parser.add_argument("--workers", type=int, default=4)
|
| 116 |
+
parser.add_argument("--seed", type=int, default=20260922)
|
| 117 |
+
parser.add_argument("--limit", type=int, default=0)
|
| 118 |
+
args = parser.parse_args()
|
| 119 |
+
if not 0 <= args.shard < args.shards:
|
| 120 |
+
raise ValueError("shard must be in [0, shards)")
|
| 121 |
+
|
| 122 |
+
checkpoint_metadata = checkpoint_provenance(args.checkpoint)
|
| 123 |
+
|
| 124 |
+
os.environ.update(METHOD_D_CHECKPOINT=str(args.checkpoint.resolve()),
|
| 125 |
+
METHOD_D_RUN_CONFIG=str(args.checkpoint_config.resolve()),
|
| 126 |
+
METHOD_D_SOURCE=str(V4 / "vla_hand_pose"))
|
| 127 |
+
from transformers import AutoTokenizer
|
| 128 |
+
from modal_build_full_editor_cache import load_backbone, apply_saved_config
|
| 129 |
+
from vla_hand_pose.v4_d_fm_0 import V4DConfig, MemoryBank, Observation, flow
|
| 130 |
+
from vla_hand_pose.dataloader import SequenceDataset, Source, make_collate, to_observations
|
| 131 |
+
from vla_hand_pose.world_model import dc_ae
|
| 132 |
+
from tactile_editor import EditorConfig, FusionConfig, TactileMethodD, Variant
|
| 133 |
+
|
| 134 |
+
torch.set_num_threads(3)
|
| 135 |
+
torch.set_float32_matmul_precision("high")
|
| 136 |
+
torch.manual_seed(args.seed)
|
| 137 |
+
device = torch.device("cuda:0")
|
| 138 |
+
backbone, config = load_backbone(device)
|
| 139 |
+
|
| 140 |
+
world_payload = torch.load(args.world_checkpoint, map_location="cpu", mmap=True, weights_only=False)
|
| 141 |
+
world_metadata = json.loads(args.world_config.read_text())
|
| 142 |
+
saved = world_payload.get("effective_config", world_metadata.get("effective_config", world_metadata.get("config")))
|
| 143 |
+
world_config = V4DConfig()
|
| 144 |
+
apply_saved_config(world_config, saved)
|
| 145 |
+
world_config.validate()
|
| 146 |
+
del world_payload
|
| 147 |
+
dataset_config = copy.deepcopy(config)
|
| 148 |
+
dataset_config.world = world_config.world
|
| 149 |
+
autoencoder = dc_ae.build(world_config.world.dcae, dtype=torch.float32).to(device)
|
| 150 |
+
fusion = FusionConfig(hidden_dim=config.expert.hidden_size, num_heads=config.expert.num_heads)
|
| 151 |
+
sampler = TactileMethodD(backbone, EditorConfig(variant=Variant.V0_BASE, fusion=fusion)).eval()
|
| 152 |
+
tokenizer = AutoTokenizer.from_pretrained(config.vlm.model_id)
|
| 153 |
+
source = Source(name="tachin", video_root="/unused",
|
| 154 |
+
annotation_file=str(args.annotations / "episode_frame_index.npz"),
|
| 155 |
+
label_folder=str(args.annotations / "episodic_annotations"),
|
| 156 |
+
jpeg_root=str(args.jpeg_root))
|
| 157 |
+
|
| 158 |
+
for split in ("train", "val", "test"):
|
| 159 |
+
rows = json.loads((args.adaptation_data / split / "rows.json").read_text())
|
| 160 |
+
identity = dict(np.load(args.common / "shared" / split / "identity.npz", allow_pickle=False))
|
| 161 |
+
if not np.array_equal(np.asarray([row["sample_id"] for row in rows]).astype(str),
|
| 162 |
+
identity["sample_id"].astype(str)):
|
| 163 |
+
raise ValueError(f"T-WLA {split} row identity mismatch")
|
| 164 |
+
all_indices = np.arange(args.shard, len(rows), args.shards, dtype=np.int64)
|
| 165 |
+
if args.limit:
|
| 166 |
+
all_indices = all_indices[:args.limit]
|
| 167 |
+
output = args.output / "bases/twla" / split / f"shard_{args.shard:02d}"
|
| 168 |
+
completion = output / ("smoke_complete.json" if args.limit else "complete.json")
|
| 169 |
+
if completion.exists():
|
| 170 |
+
print(json.dumps({"split": split, "shard": args.shard, "status": "already complete"}), flush=True)
|
| 171 |
+
continue
|
| 172 |
+
|
| 173 |
+
episodes = {str(rows[index]["episode_id"]) for index in all_indices}
|
| 174 |
+
dataset = SequenceDataset([source], cfg=dataset_config,
|
| 175 |
+
video_whitelist={"tachin": episodes}, tokenizer=tokenizer,
|
| 176 |
+
seed=0, load_images=True, training=False, deterministic=True)
|
| 177 |
+
units = {(unit.episode, unit.phase): index for index, unit in enumerate(dataset.units)}
|
| 178 |
+
plans = []
|
| 179 |
+
anchors = []
|
| 180 |
+
for common_index in all_indices:
|
| 181 |
+
row = rows[int(common_index)]
|
| 182 |
+
anchor = int(row["anchor_source_row"])
|
| 183 |
+
phase, relative = anchor % 3, anchor // 3
|
| 184 |
+
length = min(relative + 1, 9)
|
| 185 |
+
plans.append((units[(str(row["episode_id"]), phase)], relative - length + 1,
|
| 186 |
+
length, int(common_index)))
|
| 187 |
+
anchors.append(anchor)
|
| 188 |
+
dataset.set_eval_plan(plans)
|
| 189 |
+
wrapped = PlannedDataset(dataset, all_indices, anchors)
|
| 190 |
+
base_collate = make_collate(dataset_config, tokenizer)
|
| 191 |
+
def collate(items):
|
| 192 |
+
batch = base_collate(items)
|
| 193 |
+
for name in ("query_index", "future_roi", "future_roi_valid"):
|
| 194 |
+
batch[name] = torch.from_numpy(np.asarray([item[name] for item in items]))
|
| 195 |
+
return batch
|
| 196 |
+
|
| 197 |
+
count = len(all_indices)
|
| 198 |
+
fields = {
|
| 199 |
+
"base_action": ((count, 16, 192), "float32"),
|
| 200 |
+
"final_memory": ((count, 8, 960), "float16"),
|
| 201 |
+
"image_tokens": ((count, 64, 960), "float16"),
|
| 202 |
+
"clean_latent": ((count, 2, 32, 16, 16), "float16"),
|
| 203 |
+
"rgb_valid": ((count, 2), "bool"),
|
| 204 |
+
"future_roi": ((count, 2, 16, 16), "bool"),
|
| 205 |
+
"query_index": ((count,), "int64"),
|
| 206 |
+
}
|
| 207 |
+
values = {name: array(output, ("smoke_" if args.limit else "") + name, shape, dtype)
|
| 208 |
+
for name, (shape, dtype) in fields.items()}
|
| 209 |
+
marker = array(output, ("smoke_" if args.limit else "") + "row_complete", (count,), "bool")
|
| 210 |
+
remaining = np.flatnonzero(~marker)
|
| 211 |
+
loader = torch.utils.data.DataLoader(torch.utils.data.Subset(wrapped, remaining.tolist()),
|
| 212 |
+
batch_size=args.batch_size, shuffle=False, num_workers=args.workers,
|
| 213 |
+
collate_fn=collate, pin_memory=True, persistent_workers=args.workers > 0)
|
| 214 |
+
started = time.time()
|
| 215 |
+
cursor = 0
|
| 216 |
+
max_target_difference = 0.0
|
| 217 |
+
mask_mismatches = 0
|
| 218 |
+
target = np.load(args.common / "shared" / split / "target_action.npy", mmap_mode="r")
|
| 219 |
+
target_mask = np.load(args.common / "shared" / split / "action_mask.npy", mmap_mode="r")
|
| 220 |
+
with torch.inference_mode():
|
| 221 |
+
for batch_index, batch in enumerate(loader):
|
| 222 |
+
steps, step_valid, actions, action_mask = to_observations(batch, device, Observation, dtype=torch.bfloat16)
|
| 223 |
+
size = len(actions)
|
| 224 |
+
bank = MemoryBank(size, config.memory.bank_capacity, config.memory.num_slots,
|
| 225 |
+
config.memory.dim, device=device, dtype=backbone.vlm.dtype)
|
| 226 |
+
for time_index, observation in enumerate(steps[:-1]):
|
| 227 |
+
active = torch.nonzero(step_valid[:, time_index], as_tuple=True)[0]
|
| 228 |
+
if not active.numel():
|
| 229 |
+
continue
|
| 230 |
+
subset = observation if active.numel() == size else observation.index_select(active)
|
| 231 |
+
bank.append_rows(active, backbone.encode(subset, bank, rows=active))
|
| 232 |
+
final_memory, image_tokens = backbone.encode(steps[-1], bank, return_image_tokens=True)
|
| 233 |
+
generation_mask = flow.component_action_mask(size, 16, 192, config.flow, device)
|
| 234 |
+
query_indices = batch["query_index"].numpy()
|
| 235 |
+
noise = torch.stack([
|
| 236 |
+
torch.randn((16, 192), device=device, dtype=final_memory.dtype,
|
| 237 |
+
generator=torch.Generator(device=device).manual_seed(args.seed + int(index)))
|
| 238 |
+
for index in query_indices
|
| 239 |
+
])
|
| 240 |
+
normalized = sampler.sample_from_memory(
|
| 241 |
+
final_memory, generation_mask,
|
| 242 |
+
state_with_mask=(backbone.state_with_mask(steps[-1].state, steps[-1].state_mask)
|
| 243 |
+
if config.action.direct_state_to_expert else None),
|
| 244 |
+
noise=noise, num_steps=config.flow.num_inference_steps,
|
| 245 |
+
return_normalized=True,
|
| 246 |
+
).float()
|
| 247 |
+
physical = normalized * backbone.action_std.float() + backbone.action_mean.float()
|
| 248 |
+
future_images = batch["future_images"].to(device).permute(0, 1, 4, 2, 3).float().div_(127.5).sub_(1)
|
| 249 |
+
clean_latent = dc_ae.encode_targets(autoencoder, future_images, world_config.world.dcae)
|
| 250 |
+
positions = remaining[cursor:cursor + size]
|
| 251 |
+
difference = np.abs(actions.float().cpu().numpy() - target[query_indices])
|
| 252 |
+
common_mask = target_mask[query_indices]
|
| 253 |
+
max_target_difference = max(max_target_difference,
|
| 254 |
+
float(difference[common_mask].max(initial=0)))
|
| 255 |
+
mask_mismatches += int(np.count_nonzero(action_mask.cpu().numpy() != common_mask))
|
| 256 |
+
payload = {
|
| 257 |
+
"base_action": physical,
|
| 258 |
+
"final_memory": final_memory,
|
| 259 |
+
"image_tokens": image_tokens,
|
| 260 |
+
"clean_latent": clean_latent,
|
| 261 |
+
"rgb_valid": batch["future_valid"].bool() & batch["future_roi_valid"].bool(),
|
| 262 |
+
"future_roi": batch["future_roi"],
|
| 263 |
+
"query_index": batch["query_index"],
|
| 264 |
+
}
|
| 265 |
+
for name, value in payload.items():
|
| 266 |
+
if value.dtype == torch.bfloat16:
|
| 267 |
+
value = value.float()
|
| 268 |
+
result = value.detach().cpu().numpy()
|
| 269 |
+
if not np.isfinite(result).all():
|
| 270 |
+
raise ValueError(f"Nonfinite T-WLA cache field {name}")
|
| 271 |
+
values[name][positions] = result
|
| 272 |
+
values[name].flush()
|
| 273 |
+
marker[positions] = True
|
| 274 |
+
marker.flush()
|
| 275 |
+
cursor += size
|
| 276 |
+
if batch_index % 25 == 0:
|
| 277 |
+
print(json.dumps({"split": split, "shard": args.shard,
|
| 278 |
+
"done": int(marker.sum()), "total": count,
|
| 279 |
+
"rows_per_second": cursor / max(time.time() - started, 1e-6)}), flush=True)
|
| 280 |
+
if not marker.all():
|
| 281 |
+
raise RuntimeError(f"Incomplete T-WLA shard: {output}")
|
| 282 |
+
atomic_json(completion, {
|
| 283 |
+
"schema": "tachin.table1.twla_w3_cache.v1",
|
| 284 |
+
"split": split, "shard": args.shard, "shards": args.shards,
|
| 285 |
+
"rows": count, "checkpoint": str(args.checkpoint.resolve()),
|
| 286 |
+
"checkpoint_sha256": sha256(args.checkpoint), **checkpoint_metadata,
|
| 287 |
+
"memory_protocol": "up to eight available 10-Hz historical observations plus current",
|
| 288 |
+
"noise": f"per-row torch seed {args.seed} + zero-based common row index",
|
| 289 |
+
"world_checkpoint": str(args.world_checkpoint.resolve()),
|
| 290 |
+
"world_checkpoint_sha256": sha256(args.world_checkpoint),
|
| 291 |
+
"max_common_target_difference": max_target_difference,
|
| 292 |
+
"common_mask_mismatches": mask_mismatches,
|
| 293 |
+
"seconds": time.time() - started,
|
| 294 |
+
})
|
| 295 |
+
print(json.dumps({"split": split, "shard": args.shard, "status": "complete"}), flush=True)
|
| 296 |
+
|
| 297 |
+
|
| 298 |
+
if __name__ == "__main__":
|
| 299 |
+
main()
|
model/twla_tachin_common_adaptation_20260922/code/rebuild_twla_adapted_checkpoint.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Build a strict expert-only T-WLA checkpoint from an adaptation adapter.
|
| 2 |
+
|
| 3 |
+
The backbone loader casts the live model to bfloat16. Saving its complete state dict
|
| 4 |
+
therefore changes frozen FP32 tensors even when they received no optimizer updates.
|
| 5 |
+
This utility starts from the original checkpoint and replaces only the action expert
|
| 6 |
+
and the four Tachin normalization buffers. Frozen VLM/memory tensors consequently
|
| 7 |
+
remain bit-identical to the released 110k checkpoint.
|
| 8 |
+
"""
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import argparse
|
| 12 |
+
import hashlib
|
| 13 |
+
import json
|
| 14 |
+
import os
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
|
| 17 |
+
import numpy as np
|
| 18 |
+
import torch
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def sha256(path: Path) -> str:
|
| 22 |
+
digest = hashlib.sha256()
|
| 23 |
+
with path.open("rb") as stream:
|
| 24 |
+
for block in iter(lambda: stream.read(8 << 20), b""):
|
| 25 |
+
digest.update(block)
|
| 26 |
+
return digest.hexdigest()
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def main() -> None:
|
| 30 |
+
parser = argparse.ArgumentParser()
|
| 31 |
+
parser.add_argument("--initial", type=Path, required=True)
|
| 32 |
+
parser.add_argument("--adapter", type=Path, required=True)
|
| 33 |
+
parser.add_argument("--adaptation-data", type=Path, required=True)
|
| 34 |
+
parser.add_argument("--run-config", type=Path, required=True)
|
| 35 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 36 |
+
args = parser.parse_args()
|
| 37 |
+
if args.output.exists():
|
| 38 |
+
raise FileExistsError(args.output)
|
| 39 |
+
|
| 40 |
+
initial = torch.load(args.initial, map_location="cpu", mmap=True, weights_only=False)
|
| 41 |
+
adapter = torch.load(args.adapter, map_location="cpu", mmap=True, weights_only=False)
|
| 42 |
+
run_config = json.loads(args.run_config.read_text())
|
| 43 |
+
state = dict(initial["model"])
|
| 44 |
+
expert = adapter.get("expert")
|
| 45 |
+
if not isinstance(expert, dict):
|
| 46 |
+
raise ValueError("adapter checkpoint has no expert state dict")
|
| 47 |
+
expected = {name.removeprefix("expert.") for name in state if name.startswith("expert.")}
|
| 48 |
+
if set(expert) != expected:
|
| 49 |
+
raise ValueError(
|
| 50 |
+
f"expert schema mismatch: missing={sorted(expected - set(expert))[:10]}, "
|
| 51 |
+
f"extra={sorted(set(expert) - expected)[:10]}"
|
| 52 |
+
)
|
| 53 |
+
changed = 0
|
| 54 |
+
for short_name, value in expert.items():
|
| 55 |
+
name = f"expert.{short_name}"
|
| 56 |
+
replacement = value.to(dtype=state[name].dtype)
|
| 57 |
+
changed += int(not torch.equal(state[name], replacement))
|
| 58 |
+
state[name] = replacement
|
| 59 |
+
if changed == 0:
|
| 60 |
+
raise ValueError("adapted action expert is identical to the initial expert")
|
| 61 |
+
|
| 62 |
+
statistics = json.loads((args.adaptation_data / "statistics.json").read_text())["keypoints"]
|
| 63 |
+
for name in ("action_mean", "action_std", "state_mean", "state_std"):
|
| 64 |
+
value = torch.as_tensor(np.asarray(statistics[name]), dtype=state[name].dtype)
|
| 65 |
+
if tuple(value.shape) != tuple(state[name].shape):
|
| 66 |
+
raise ValueError(f"normalization shape mismatch for {name}")
|
| 67 |
+
state[name] = value
|
| 68 |
+
state["stats_loaded"] = torch.ones_like(state["stats_loaded"], dtype=torch.bool)
|
| 69 |
+
|
| 70 |
+
payload = {
|
| 71 |
+
"model": state,
|
| 72 |
+
"step": int(adapter["step"]),
|
| 73 |
+
"epoch": -1,
|
| 74 |
+
"best_val": float(adapter["val_loss"]),
|
| 75 |
+
"adaptation": {
|
| 76 |
+
**run_config,
|
| 77 |
+
"assembly": "original frozen state + adapted expert + Tachin normalization",
|
| 78 |
+
"initial_sha256": sha256(args.initial),
|
| 79 |
+
"adapter_sha256": sha256(args.adapter),
|
| 80 |
+
"frozen_tensor_contract": "vlm.* and memory.* bit-identical to initial",
|
| 81 |
+
},
|
| 82 |
+
}
|
| 83 |
+
temporary = args.output.with_suffix(args.output.suffix + ".tmp")
|
| 84 |
+
torch.save(payload, temporary)
|
| 85 |
+
os.replace(temporary, args.output)
|
| 86 |
+
print(json.dumps({
|
| 87 |
+
"output": str(args.output),
|
| 88 |
+
"step": payload["step"],
|
| 89 |
+
"best_val": payload["best_val"],
|
| 90 |
+
"changed_expert_tensors": changed,
|
| 91 |
+
"sha256": sha256(args.output),
|
| 92 |
+
}, indent=2))
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
if __name__ == "__main__":
|
| 96 |
+
main()
|
model/twla_tachin_common_adaptation_20260922/code/verify_twla_tachin_adaptation.py
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Audit the expert-only T-WLA Tachin adaptation before cache generation."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import argparse
|
| 5 |
+
import hashlib
|
| 6 |
+
import json
|
| 7 |
+
import math
|
| 8 |
+
import os
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
import numpy as np
|
| 12 |
+
import torch
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def sha256(path: Path) -> str:
|
| 16 |
+
value = hashlib.sha256()
|
| 17 |
+
with path.open("rb") as stream:
|
| 18 |
+
for block in iter(lambda: stream.read(8 << 20), b""):
|
| 19 |
+
value.update(block)
|
| 20 |
+
return value.hexdigest()
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def atomic_json(path: Path, payload) -> None:
|
| 24 |
+
temporary = path.with_suffix(path.suffix + ".tmp")
|
| 25 |
+
temporary.write_text(json.dumps(payload, indent=2, default=str) + "\n")
|
| 26 |
+
os.replace(temporary, path)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def main() -> None:
|
| 30 |
+
parser = argparse.ArgumentParser()
|
| 31 |
+
parser.add_argument("--run", type=Path, required=True)
|
| 32 |
+
parser.add_argument("--initial", type=Path, required=True)
|
| 33 |
+
parser.add_argument("--adaptation-data", type=Path, required=True)
|
| 34 |
+
parser.add_argument("--common", type=Path, required=True)
|
| 35 |
+
args = parser.parse_args()
|
| 36 |
+
|
| 37 |
+
required = ("COMPLETE.json", "config.json", "metrics.jsonl", "best.pt")
|
| 38 |
+
missing = [name for name in required if not (args.run / name).is_file()]
|
| 39 |
+
if missing:
|
| 40 |
+
raise FileNotFoundError(f"Incomplete adaptation run; missing {missing}")
|
| 41 |
+
|
| 42 |
+
completion = json.loads((args.run / "COMPLETE.json").read_text())
|
| 43 |
+
config = json.loads((args.run / "config.json").read_text())
|
| 44 |
+
metrics = [json.loads(line) for line in (args.run / "metrics.jsonl").read_text().splitlines()
|
| 45 |
+
if line.strip()]
|
| 46 |
+
if not metrics or int(metrics[0].get("step", -1)) != 0 or "val_loss" not in metrics[0]:
|
| 47 |
+
raise ValueError("metrics.jsonl does not begin with the step-0 validation")
|
| 48 |
+
initial_validation = float(metrics[0]["val_loss"])
|
| 49 |
+
selected_step = int(completion["selected_step"])
|
| 50 |
+
best_validation = float(completion["best_validation_loss"])
|
| 51 |
+
if selected_step <= 0:
|
| 52 |
+
raise ValueError("Tachin adaptation did not beat the unadapted step-0 checkpoint")
|
| 53 |
+
if not (math.isfinite(best_validation) and best_validation < initial_validation):
|
| 54 |
+
raise ValueError(
|
| 55 |
+
f"Selected validation {best_validation} did not improve step 0 {initial_validation}")
|
| 56 |
+
if completion.get("test_used") is not False:
|
| 57 |
+
raise ValueError("Adaptation completion does not certify test_used=false")
|
| 58 |
+
|
| 59 |
+
initial_sha = sha256(args.initial)
|
| 60 |
+
common_sha = sha256(args.common / "complete.json")
|
| 61 |
+
if completion.get("initial_sha256") != initial_sha:
|
| 62 |
+
raise ValueError("COMPLETE.json initial checkpoint hash mismatch")
|
| 63 |
+
if config.get("initial_sha256") != initial_sha:
|
| 64 |
+
raise ValueError("config.json initial checkpoint hash mismatch")
|
| 65 |
+
if config.get("common_complete_sha256") != common_sha:
|
| 66 |
+
raise ValueError("config.json common-cache hash mismatch")
|
| 67 |
+
|
| 68 |
+
adapted = torch.load(args.run / "best.pt", map_location="cpu", mmap=True,
|
| 69 |
+
weights_only=False)
|
| 70 |
+
initial = torch.load(args.initial, map_location="cpu", mmap=True, weights_only=False)
|
| 71 |
+
if int(adapted.get("step", -1)) != selected_step:
|
| 72 |
+
raise ValueError("best.pt selected step disagrees with COMPLETE.json")
|
| 73 |
+
if not isinstance(adapted.get("adaptation"), dict):
|
| 74 |
+
raise ValueError("best.pt has no adaptation provenance")
|
| 75 |
+
adapted_state = adapted["model"]
|
| 76 |
+
initial_state = initial["model"]
|
| 77 |
+
if set(adapted_state) != set(initial_state):
|
| 78 |
+
missing_keys = sorted(set(initial_state) - set(adapted_state))[:10]
|
| 79 |
+
extra_keys = sorted(set(adapted_state) - set(initial_state))[:10]
|
| 80 |
+
raise ValueError(f"State schema changed; missing={missing_keys}, extra={extra_keys}")
|
| 81 |
+
|
| 82 |
+
frozen_prefixes = ("vlm.", "memory.")
|
| 83 |
+
frozen_tensors = 0
|
| 84 |
+
for name in initial_state:
|
| 85 |
+
if not name.startswith(frozen_prefixes):
|
| 86 |
+
continue
|
| 87 |
+
if not torch.equal(initial_state[name], adapted_state[name]):
|
| 88 |
+
delta = float((initial_state[name].float() - adapted_state[name].float()).abs().max())
|
| 89 |
+
raise ValueError(f"Frozen tensor changed: {name}, max delta {delta}")
|
| 90 |
+
frozen_tensors += 1
|
| 91 |
+
|
| 92 |
+
expert_changed = []
|
| 93 |
+
for name in initial_state:
|
| 94 |
+
if name.startswith("expert.") and not torch.equal(initial_state[name], adapted_state[name]):
|
| 95 |
+
expert_changed.append(name)
|
| 96 |
+
if not expert_changed:
|
| 97 |
+
raise ValueError("No action-expert tensor changed during adaptation")
|
| 98 |
+
|
| 99 |
+
stats = json.loads((args.adaptation_data / "statistics.json").read_text())["keypoints"]
|
| 100 |
+
for name in ("action_mean", "action_std", "state_mean", "state_std"):
|
| 101 |
+
actual = adapted_state[name].float().numpy()
|
| 102 |
+
expected = np.asarray(stats[name], dtype=np.float32)
|
| 103 |
+
if not np.allclose(actual, expected, rtol=1e-6, atol=1e-7):
|
| 104 |
+
raise ValueError(f"Adapted checkpoint {name} does not match Tachin statistics")
|
| 105 |
+
if not bool(adapted_state["stats_loaded"]):
|
| 106 |
+
raise ValueError("Adapted checkpoint statistics are not marked loaded")
|
| 107 |
+
|
| 108 |
+
report = {
|
| 109 |
+
"status": "passed",
|
| 110 |
+
"run": str(args.run.resolve()),
|
| 111 |
+
"selected_step": selected_step,
|
| 112 |
+
"initial_validation_loss": initial_validation,
|
| 113 |
+
"best_validation_loss": best_validation,
|
| 114 |
+
"validation_reduction": initial_validation - best_validation,
|
| 115 |
+
"test_used": False,
|
| 116 |
+
"initial_checkpoint_sha256": initial_sha,
|
| 117 |
+
"adapted_checkpoint_sha256": sha256(args.run / "best.pt"),
|
| 118 |
+
"common_complete_sha256": common_sha,
|
| 119 |
+
"frozen_vlm_memory_tensors_verified": frozen_tensors,
|
| 120 |
+
"changed_expert_tensors": len(expert_changed),
|
| 121 |
+
"normalization": "Tachin train-only keypoints statistics",
|
| 122 |
+
}
|
| 123 |
+
atomic_json(args.run / "verification.json", report)
|
| 124 |
+
print(json.dumps(report, indent=2))
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
if __name__ == "__main__":
|
| 128 |
+
main()
|
model/twla_tachin_common_adaptation_20260922/provenance/COMPLETE.json
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema": "twla.tachin_common_adaptation.v1",
|
| 3 |
+
"selected_step": 800,
|
| 4 |
+
"best_validation_loss": 0.5552965396678234,
|
| 5 |
+
"initial_checkpoint": "/scratch/hannahyao24/VLA-tactile_paper/checkpoints/v4d_ten_stageA/best.pt",
|
| 6 |
+
"initial_sha256": "ccd65fcf845ec01c7de46bb76e9cf0fee33674dd848771e114ff23b2535f13d9",
|
| 7 |
+
"checkpoint_sha256": "eb6ef381882dfbf4188d2bce13f3ea470214f037b0d3340e6aadb6064cf90ba5",
|
| 8 |
+
"test_used": false,
|
| 9 |
+
"trainable_parameters": 102968832,
|
| 10 |
+
"frozen_parameters": 325414726,
|
| 11 |
+
"seconds": 1778.7159943580627
|
| 12 |
+
}
|
model/twla_tachin_common_adaptation_20260922/provenance/config.json
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"checkpoint": "checkpoints/v4d_ten_stageA/best.pt",
|
| 3 |
+
"checkpoint_config": "checkpoints/v4d_ten_stageA/config_used.json",
|
| 4 |
+
"annotations": "artifacts/modal/v4-data/tachin_ours",
|
| 5 |
+
"jpeg_root": "artifacts/modal/v4-data/tachin_jpeg",
|
| 6 |
+
"adaptation_data": "artifacts/datasets/tachin_adaptation_20260922",
|
| 7 |
+
"common": "artifacts/tachin_table1_20260922",
|
| 8 |
+
"output": "runs/twla_tachin_common_adaptation_20260922",
|
| 9 |
+
"steps": 1000,
|
| 10 |
+
"batch_size": 32,
|
| 11 |
+
"accumulation": 4,
|
| 12 |
+
"lr": 5e-05,
|
| 13 |
+
"warmup": 100,
|
| 14 |
+
"val_every": 100,
|
| 15 |
+
"workers": 8,
|
| 16 |
+
"seed": 20260922,
|
| 17 |
+
"smoke": false,
|
| 18 |
+
"protocol": "Tachin-only expert adaptation from T-WLA-110k",
|
| 19 |
+
"targets": "exact common cache target_action/action_mask",
|
| 20 |
+
"normalization": "exact common Tachin train-only keypoint statistics",
|
| 21 |
+
"history": "up to eight prior 10-Hz observations plus current",
|
| 22 |
+
"frozen": [
|
| 23 |
+
"vlm",
|
| 24 |
+
"memory"
|
| 25 |
+
],
|
| 26 |
+
"trainable": [
|
| 27 |
+
"expert"
|
| 28 |
+
],
|
| 29 |
+
"train_rows": 18208,
|
| 30 |
+
"val_rows": 1006,
|
| 31 |
+
"initial_sha256": "ccd65fcf845ec01c7de46bb76e9cf0fee33674dd848771e114ff23b2535f13d9",
|
| 32 |
+
"common_complete_sha256": "1cffe026d12e556cc2f66fbda0b34a65a985f50d7aeb0a5adb50571e8437e740"
|
| 33 |
+
}
|
model/twla_tachin_common_adaptation_20260922/provenance/initial_config_used.json
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_comment": "Variant D on all ten domains. Validation does NOT run in this process: `val_every` is set so it never fires, and `snapshot_for_validator` makes each save also drop `snapshots/snap_<step>.pt` for `validate_loop` to score on its own GPU. Measured 2026-09-08: an in-loop pass cost 25 min against 73 min of training, 34% of the wall clock, because the validation frames go cold between passes. `best.pt` is written by the validator, not here.\\n\\nThe per-domain `weight` is NOT the share -- it is the share divided by the domain's natural size, because MixtureSampler and compute_stats both consume `weight * units`. Shares were set by the user on 2026-09-07 as relative parts against epic30 (30/30/15/15/5/5/5/3/2/2); see final/README.md. CHANGING A SHARE MEANS RECOMPUTING THE WEIGHT.",
|
| 3 |
+
"variant": "d",
|
| 4 |
+
"seed": 0,
|
| 5 |
+
"batch_size": 64,
|
| 6 |
+
"num_workers": 12,
|
| 7 |
+
"prefetch_factor": 2,
|
| 8 |
+
"max_steps": 110000,
|
| 9 |
+
"log_every": 50,
|
| 10 |
+
"val_every": 110000,
|
| 11 |
+
"save_every": 2000,
|
| 12 |
+
"output_dir": "/runs/v4d_ten_stageA",
|
| 13 |
+
"data": {
|
| 14 |
+
"statistics": "/data/splits/ten_v3/stats_80hz.json",
|
| 15 |
+
"sources": [
|
| 16 |
+
{
|
| 17 |
+
"name": "epic30",
|
| 18 |
+
"video_root": "/data/epic_vitra/Video/Epic-Kitchen_root",
|
| 19 |
+
"annotation_file": "/data/epic30_ours_v2/episode_frame_index.npz",
|
| 20 |
+
"label_folder": "/data/epic30_ours_v2",
|
| 21 |
+
"jpeg_root": "/data/preprocessed/epic30_ours_v2_jpeg",
|
| 22 |
+
"split_dir": "/data/splits/epic30_v5",
|
| 23 |
+
"weight": 0.2654
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"name": "egotouch",
|
| 27 |
+
"video_root": "/data/et_video",
|
| 28 |
+
"annotation_file": "/data/egotouch_ours/episode_frame_index.npz",
|
| 29 |
+
"label_folder": "/data/egotouch_ours/episodic_annotations",
|
| 30 |
+
"jpeg_root": "/data/preprocessed/egotouch_jpeg",
|
| 31 |
+
"split_dir": "/data/splits/egotouch_v2",
|
| 32 |
+
"weight": 0.3694
|
| 33 |
+
},
|
| 34 |
+
{
|
| 35 |
+
"name": "gigahands",
|
| 36 |
+
"video_root": "/data/gigahands_vitra_mano_strict_v1__CHONGHE/Video/GigaHands_root",
|
| 37 |
+
"annotation_file": "/data/gigahands_ours/episode_frame_index.npz",
|
| 38 |
+
"label_folder": "/data/gigahands_ours/episodic_annotations",
|
| 39 |
+
"jpeg_root": "/data/preprocessed/gigahands_jpeg",
|
| 40 |
+
"split_dir": "/data/splits/gigahands_v1",
|
| 41 |
+
"weight": 0.2811
|
| 42 |
+
},
|
| 43 |
+
{
|
| 44 |
+
"name": "egoexo4d",
|
| 45 |
+
"video_root": "/data/egoexo4d_OFFICIAL/undistorted",
|
| 46 |
+
"annotation_file": "/data/vitra1m_egoexo4d/egoexo4d/episode_frame_index.npz",
|
| 47 |
+
"label_folder": "/data/vitra1m_egoexo4d/egoexo4d/episodic_annotations",
|
| 48 |
+
"jpeg_root": "/data/preprocessed/egoexo4d_jpeg",
|
| 49 |
+
"split_dir": "/data/splits/egoexo4d_v1",
|
| 50 |
+
"weight": 0.2962
|
| 51 |
+
},
|
| 52 |
+
{
|
| 53 |
+
"name": "oakink2",
|
| 54 |
+
"video_root": "/data/oakink2_vitra_streaming_v1__CHONGHE/Video/OakInk2_root",
|
| 55 |
+
"annotation_file": "/data/oakink2_ours/episode_frame_index.npz",
|
| 56 |
+
"label_folder": "/data/oakink2_ours/episodic_annotations",
|
| 57 |
+
"jpeg_root": "/data/preprocessed/oakink2_jpeg",
|
| 58 |
+
"split_dir": "/data/splits/oakink2_v2",
|
| 59 |
+
"weight": 0.2275
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"name": "taco",
|
| 63 |
+
"video_root": "/data/taco_raw/v1/Egocentric_RGB_Videos_x/Egocentric_RGB_Videos",
|
| 64 |
+
"annotation_file": "/data/taco_ours/episode_frame_index.npz",
|
| 65 |
+
"label_folder": "/data/taco_ours/episodic_annotations",
|
| 66 |
+
"jpeg_root": "/data/preprocessed/taco_jpeg",
|
| 67 |
+
"split_dir": "/data/splits/taco_v1",
|
| 68 |
+
"weight": 0.278
|
| 69 |
+
},
|
| 70 |
+
{
|
| 71 |
+
"name": "hot3d",
|
| 72 |
+
"video_root": "/data/hot3d_vitra_atomic_v1__CHONGHE/Video/HOT3D_root",
|
| 73 |
+
"annotation_file": "/data/hot3d_ours/episode_frame_index.npz",
|
| 74 |
+
"label_folder": "/data/hot3d_ours/episodic_annotations",
|
| 75 |
+
"jpeg_root": "/data/preprocessed/hot3d_jpeg",
|
| 76 |
+
"split_dir": "/data/splits/hot3d_v2",
|
| 77 |
+
"weight": 0.3513
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"name": "arctic",
|
| 81 |
+
"video_root": "/data/arctic_vitra/Video/arctic_root",
|
| 82 |
+
"annotation_file": "/data/arctic_vitra/arctic/episode_frame_index.npz",
|
| 83 |
+
"label_folder": "/data/arctic_vitra/arctic/episodic_annotations",
|
| 84 |
+
"jpeg_root": "/data/preprocessed/arctic_jpeg",
|
| 85 |
+
"split_dir": "/data/splits/arctic_v1",
|
| 86 |
+
"weight": 0.3142
|
| 87 |
+
},
|
| 88 |
+
{
|
| 89 |
+
"name": "h2o",
|
| 90 |
+
"video_root": "/data/h2o_vitra/Video/h2o_root",
|
| 91 |
+
"annotation_file": "/data/h2o_ours/episode_frame_index.npz",
|
| 92 |
+
"label_folder": "/data/h2o_ours/episodic_annotations",
|
| 93 |
+
"jpeg_root": "/data/preprocessed/h2o_jpeg",
|
| 94 |
+
"split_dir": "/data/splits/h2o_v2",
|
| 95 |
+
"weight": 0.4535
|
| 96 |
+
},
|
| 97 |
+
{
|
| 98 |
+
"name": "tachin",
|
| 99 |
+
"video_root": "/data/tachin_precut_v1__CHONGHE",
|
| 100 |
+
"annotation_file": "/data/tachin_ours/episode_frame_index.npz",
|
| 101 |
+
"label_folder": "/data/tachin_ours/episodic_annotations",
|
| 102 |
+
"jpeg_root": "/data/preprocessed/tachin_jpeg",
|
| 103 |
+
"split_dir": "/data/splits/tachin_v2",
|
| 104 |
+
"weight": 1.0
|
| 105 |
+
}
|
| 106 |
+
]
|
| 107 |
+
},
|
| 108 |
+
"config": {
|
| 109 |
+
"optim": {
|
| 110 |
+
"warmup_steps": 1000,
|
| 111 |
+
"decay_steps": 110000
|
| 112 |
+
}
|
| 113 |
+
},
|
| 114 |
+
"snapshot_for_validator": true
|
| 115 |
+
}
|
model/twla_tachin_common_adaptation_20260922/provenance/initial_ten_domain_config.json
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_comment": "Variant D on all ten domains. Validation does NOT run in this process: `val_every` is set so it never fires, and `snapshot_for_validator` makes each save also drop `snapshots/snap_<step>.pt` for `validate_loop` to score on its own GPU. Measured 2026-09-08: an in-loop pass cost 25 min against 73 min of training, 34% of the wall clock, because the validation frames go cold between passes. `best.pt` is written by the validator, not here.\\n\\nThe per-domain `weight` is NOT the share -- it is the share divided by the domain's natural size, because MixtureSampler and compute_stats both consume `weight * units`. Shares were set by the user on 2026-09-07 as relative parts against epic30 (30/30/15/15/5/5/5/3/2/2); see final/README.md. CHANGING A SHARE MEANS RECOMPUTING THE WEIGHT.",
|
| 3 |
+
"variant": "d",
|
| 4 |
+
"seed": 0,
|
| 5 |
+
"batch_size": 64,
|
| 6 |
+
"num_workers": 12,
|
| 7 |
+
"prefetch_factor": 2,
|
| 8 |
+
"max_steps": 110000,
|
| 9 |
+
"log_every": 50,
|
| 10 |
+
"val_every": 110000,
|
| 11 |
+
"save_every": 2000,
|
| 12 |
+
"output_dir": "/runs/v4d_ten_stageA",
|
| 13 |
+
"data": {
|
| 14 |
+
"statistics": "/data/splits/ten_v3/stats_80hz.json",
|
| 15 |
+
"sources": [
|
| 16 |
+
{
|
| 17 |
+
"name": "epic30",
|
| 18 |
+
"video_root": "/data/epic_vitra/Video/Epic-Kitchen_root",
|
| 19 |
+
"annotation_file": "/data/epic30_ours_v2/episode_frame_index.npz",
|
| 20 |
+
"label_folder": "/data/epic30_ours_v2",
|
| 21 |
+
"jpeg_root": "/data/preprocessed/epic30_ours_v2_jpeg",
|
| 22 |
+
"split_dir": "/data/splits/epic30_v5",
|
| 23 |
+
"weight": 0.2654
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"name": "egotouch",
|
| 27 |
+
"video_root": "/data/et_video",
|
| 28 |
+
"annotation_file": "/data/egotouch_ours/episode_frame_index.npz",
|
| 29 |
+
"label_folder": "/data/egotouch_ours/episodic_annotations",
|
| 30 |
+
"jpeg_root": "/data/preprocessed/egotouch_jpeg",
|
| 31 |
+
"split_dir": "/data/splits/egotouch_v2",
|
| 32 |
+
"weight": 0.3694
|
| 33 |
+
},
|
| 34 |
+
{
|
| 35 |
+
"name": "gigahands",
|
| 36 |
+
"video_root": "/data/gigahands_vitra_mano_strict_v1__CHONGHE/Video/GigaHands_root",
|
| 37 |
+
"annotation_file": "/data/gigahands_ours/episode_frame_index.npz",
|
| 38 |
+
"label_folder": "/data/gigahands_ours/episodic_annotations",
|
| 39 |
+
"jpeg_root": "/data/preprocessed/gigahands_jpeg",
|
| 40 |
+
"split_dir": "/data/splits/gigahands_v1",
|
| 41 |
+
"weight": 0.2811
|
| 42 |
+
},
|
| 43 |
+
{
|
| 44 |
+
"name": "egoexo4d",
|
| 45 |
+
"video_root": "/data/egoexo4d_OFFICIAL/undistorted",
|
| 46 |
+
"annotation_file": "/data/vitra1m_egoexo4d/egoexo4d/episode_frame_index.npz",
|
| 47 |
+
"label_folder": "/data/vitra1m_egoexo4d/egoexo4d/episodic_annotations",
|
| 48 |
+
"jpeg_root": "/data/preprocessed/egoexo4d_jpeg",
|
| 49 |
+
"split_dir": "/data/splits/egoexo4d_v1",
|
| 50 |
+
"weight": 0.2962
|
| 51 |
+
},
|
| 52 |
+
{
|
| 53 |
+
"name": "oakink2",
|
| 54 |
+
"video_root": "/data/oakink2_vitra_streaming_v1__CHONGHE/Video/OakInk2_root",
|
| 55 |
+
"annotation_file": "/data/oakink2_ours/episode_frame_index.npz",
|
| 56 |
+
"label_folder": "/data/oakink2_ours/episodic_annotations",
|
| 57 |
+
"jpeg_root": "/data/preprocessed/oakink2_jpeg",
|
| 58 |
+
"split_dir": "/data/splits/oakink2_v2",
|
| 59 |
+
"weight": 0.2275
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"name": "taco",
|
| 63 |
+
"video_root": "/data/taco_raw/v1/Egocentric_RGB_Videos_x/Egocentric_RGB_Videos",
|
| 64 |
+
"annotation_file": "/data/taco_ours/episode_frame_index.npz",
|
| 65 |
+
"label_folder": "/data/taco_ours/episodic_annotations",
|
| 66 |
+
"jpeg_root": "/data/preprocessed/taco_jpeg",
|
| 67 |
+
"split_dir": "/data/splits/taco_v1",
|
| 68 |
+
"weight": 0.278
|
| 69 |
+
},
|
| 70 |
+
{
|
| 71 |
+
"name": "hot3d",
|
| 72 |
+
"video_root": "/data/hot3d_vitra_atomic_v1__CHONGHE/Video/HOT3D_root",
|
| 73 |
+
"annotation_file": "/data/hot3d_ours/episode_frame_index.npz",
|
| 74 |
+
"label_folder": "/data/hot3d_ours/episodic_annotations",
|
| 75 |
+
"jpeg_root": "/data/preprocessed/hot3d_jpeg",
|
| 76 |
+
"split_dir": "/data/splits/hot3d_v2",
|
| 77 |
+
"weight": 0.3513
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"name": "arctic",
|
| 81 |
+
"video_root": "/data/arctic_vitra/Video/arctic_root",
|
| 82 |
+
"annotation_file": "/data/arctic_vitra/arctic/episode_frame_index.npz",
|
| 83 |
+
"label_folder": "/data/arctic_vitra/arctic/episodic_annotations",
|
| 84 |
+
"jpeg_root": "/data/preprocessed/arctic_jpeg",
|
| 85 |
+
"split_dir": "/data/splits/arctic_v1",
|
| 86 |
+
"weight": 0.3142
|
| 87 |
+
},
|
| 88 |
+
{
|
| 89 |
+
"name": "h2o",
|
| 90 |
+
"video_root": "/data/h2o_vitra/Video/h2o_root",
|
| 91 |
+
"annotation_file": "/data/h2o_ours/episode_frame_index.npz",
|
| 92 |
+
"label_folder": "/data/h2o_ours/episodic_annotations",
|
| 93 |
+
"jpeg_root": "/data/preprocessed/h2o_jpeg",
|
| 94 |
+
"split_dir": "/data/splits/h2o_v2",
|
| 95 |
+
"weight": 0.4535
|
| 96 |
+
},
|
| 97 |
+
{
|
| 98 |
+
"name": "tachin",
|
| 99 |
+
"video_root": "/data/tachin_precut_v1__CHONGHE",
|
| 100 |
+
"annotation_file": "/data/tachin_ours/episode_frame_index.npz",
|
| 101 |
+
"label_folder": "/data/tachin_ours/episodic_annotations",
|
| 102 |
+
"jpeg_root": "/data/preprocessed/tachin_jpeg",
|
| 103 |
+
"split_dir": "/data/splits/tachin_v2",
|
| 104 |
+
"weight": 1.0
|
| 105 |
+
}
|
| 106 |
+
]
|
| 107 |
+
},
|
| 108 |
+
"config": {
|
| 109 |
+
"optim": {
|
| 110 |
+
"warmup_steps": 1000,
|
| 111 |
+
"decay_steps": 110000
|
| 112 |
+
}
|
| 113 |
+
},
|
| 114 |
+
"snapshot_for_validator": true
|
| 115 |
+
}
|
model/twla_tachin_common_adaptation_20260922/provenance/metrics.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
model/twla_tachin_common_adaptation_20260922/provenance/verification.json
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "passed",
|
| 3 |
+
"run": "/scratch/hannahyao24/VLA-tactile_paper/runs/twla_tachin_common_adaptation_20260922",
|
| 4 |
+
"selected_step": 800,
|
| 5 |
+
"initial_validation_loss": 0.7941877830099633,
|
| 6 |
+
"best_validation_loss": 0.5552965396678234,
|
| 7 |
+
"validation_reduction": 0.23889124334213985,
|
| 8 |
+
"test_used": false,
|
| 9 |
+
"initial_checkpoint_sha256": "ccd65fcf845ec01c7de46bb76e9cf0fee33674dd848771e114ff23b2535f13d9",
|
| 10 |
+
"adapted_checkpoint_sha256": "8df6c87fda2e5ec09d61686451744c37bdcc2d1b4d9d13b58893851be620db7d",
|
| 11 |
+
"common_complete_sha256": "1cffe026d12e556cc2f66fbda0b34a65a985f50d7aeb0a5adb50571e8437e740",
|
| 12 |
+
"frozen_vlm_memory_tensors_verified": 381,
|
| 13 |
+
"changed_expert_tensors": 155,
|
| 14 |
+
"normalization": "Tachin train-only keypoints statistics"
|
| 15 |
+
}
|